Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
"""Source citations shared by agent backends.
Some model backends answer with opaque inline citation markers instead of
links. Codex (the OpenAI app-server) emits
U+E200 "cite" U+E202 <ref_id> [U+E202 <ref_id> ...] U+E201
where each ``ref_id`` names one result of a web search performed earlier in the
same native thread. The results themselves arrive in separate notifications, so
a backend harvests them into a map scoped to that thread and hands the map here
when a message is about to be emitted.
This module owns the shared half of the feature: the marker grammar, the
sanitizing of untrusted titles and URLs, and the rewrite into ordinary Markdown
links plus a structured sidecar. It is deliberately transport-agnostic - the
Markdown text is what every IM platform delivers, and the sidecar is what the
Web renderer upgrades into compact citation badges.
Two rules shape every branch below: a citation we cannot resolve is *downgraded*
to a visible label, never silently dropped, and a URL is never invented from a
ref_id or from search order.
"""
from __future__ import annotations
import hashlib
import re
import secrets
from dataclasses import dataclass
from html.entities import html5 as HTML5_ENTITIES
from typing import Any, Callable, Iterable, Mapping, Optional, Protocol
from urllib.parse import unquote
import idna
from core.reply_enhancer import (
MarkdownUnit,
markdown_link_units,
mask_citation_slots,
mask_hidden_and_code,
percent_encode,
spell_uri,
)
# The private-use delimiters the marker is wrapped in.
_START = "\ue200"
_SEP = "\ue202"
_END = "\ue201"
# The private-use delimiters an internal registration token is wrapped in.
# Deliberately not the marker's own: a token is this module's private handle on
# one registered marker, and nothing downstream may read it as citation grammar.
_TOKEN_OPEN = "\ue010"
_TOKEN_CLOSE = "\ue011"
_TOKEN_NONCE_BYTES = 16
_TOKEN_RE = re.compile(
f"{_TOKEN_OPEN}[0-9a-f]{{{_TOKEN_NONCE_BYTES * 2}}}{_TOKEN_CLOSE}"
)
# One complete marker. The payload may not contain a start/end delimiter, so an
# unterminated marker never matches: a stream that split a marker across chunks
# keeps its raw text instead of swallowing the answer around it. Requiring the
# literal ``cite`` keyword also keeps sibling U+E200 markers (``visualize``)
# untouched.
CITATION_MARKER_RE = re.compile(f"{_START}cite{_SEP}([^{_START}{_END}]*){_END}")
# A marker and the token it is registered as are both this module's own text:
# one is replaced by the citation it asked for, the other by the marker it
# stands for, and neither reaches a reader. So neither is part of a reference
# identifier the Markdown parser matches against a definition, and the reply
# parser is told so wherever it is asked where a citation may be written.
_SLOT_TRANSPARENT_RE = re.compile(
f"{CITATION_MARKER_RE.pattern}|{_TOKEN_RE.pattern}"
)
# Real ref_ids look like ``turn0view1``. Keep the shape permissive but bounded so
# a malformed payload falls through to the unresolved fallback.
_REF_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,64}")
_ALLOWED_SCHEMES = frozenset({"http", "https"})
# C0/C1 controls, line/paragraph separators, and the whole private-use area
# (which is where the markers themselves live), plus the invisible formatting
# characters a single-line source title has no legitimate use for: the bidi
# marks, embeddings, overrides and isolates that let a title render its own
# text backwards, and the zero-width space, word joiner and byte order mark.
#
# ZWNJ, ZWJ and the variation selectors are deliberately left in: they carry
# meaning inside a real title, joining an emoji sequence or spelling an Indic
# or Persian word, and none of them can reorder what the reader sees.
_CONTROL_RE = re.compile(
"[\x00-\x1f\x7f-\x9f\u061c\u200b\u200e\u200f\u202a-\u202e"
"\u2028\u2029\u2060\u2066-\u2069\ufeff\ue000-\uf8ff]"
)
# What WHATWG's URL parser deletes from its input before parsing anything: a tab
# or newline anywhere, and C0 controls or spaces at either end.
_URL_REMOVED_RE = re.compile(r"[\t\n\r]")
_URL_TRIMMED = "".join(chr(code) for code in range(0x21))
_DECIMAL_DIGITS = frozenset("0123456789")
_OCTAL_DIGITS = frozenset("01234567")
_HEX_DIGITS = frozenset("0123456789abcdefABCDEF")
# The widest an IPv4 number can be written per radix and still fit in 32 bits
# (``4294967295``, ``0xffffffff``, ``0o37777777777``), and the stand-in for one
# written wider. Every check that consumes a number rejects a value this big, so
# the exact magnitude past the cap never matters - and not converting it keeps
# CPython's bound on decimal ``int`` conversion out of an untrusted host.
_IPV4_WIDEST = {10: 10, 16: 8, 8: 11}
_IPV4_TOO_LARGE = 1 << 32
# A CommonMark character reference, which is what a link destination resolves
# before it becomes an href. Providers that lifted a URL out of HTML hand over
# ``&`` for ``&``, so one round of this is applied on the way in - and the
# same grammar decides, on the way out, which ``&`` has to be escaped so the
# renderer cannot resolve it a second time.
_REFERENCE_RE = re.compile(
r"&(#[0-9]{1,7}|#[Xx][0-9A-Fa-f]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});"
)
# What a citation destination has to spell that plain URI spelling does not:
# an ``&`` that starts a live reference would be resolved a second time on the
# way out, turning one delivered URL into a different one. It is written as an
# escape BEFORE ``spell_uri`` runs, which keeps it untouched - so the result is
# still a fixed point, and the reference grammar stays defined once. A
# parenthesis is not here: it is part of the address, and escaping it in the URL
# names a different page. Only the link needs it guarded - see
# ``_link_spelling``.
_CITATION_UNSAFE_RE = re.compile(rf"&(?={_REFERENCE_RE.pattern[1:]})")
# The two characters of an address that can end a Markdown destination early.
_DESTINATION_PAREN_RE = re.compile(r"[()]")
# WHATWG forbidden domain code points, checked after percent-decoding. The C0
# range is there in full: a host is not allowed to hold any of it, and a
# citation whose host carries one names a page no browser opens.
_FORBIDDEN_DOMAIN = frozenset(
"".join(chr(code) for code in range(0x20)) + " #%/:<>?@[\\]^|\x7f"
)
# The ports a browser drops from the URL it shows, because they are the ones it
# would have used anyway.
_DEFAULT_PORTS = {"http": 80, "https": 443}
_PORT_MAX = 65535
_PORT_DIGITS = len(str(_PORT_MAX))
# An IPv6 host is wrapped in brackets, which ``_canonical_uri`` percent-encodes
# along with every other character Markdown would read as syntax - so after
# spelling, a host the provider wrote ``[::1]`` and a host it wrote ``%5B::1%5D``
# are the same string. These patterns therefore recognize the wrapper in either
# spelling, and say nothing about which one the destination was written in:
# ``_authority_brackets_literal`` answers that, from the destination as Markdown
# resolved it, BEFORE spelling erased the difference.
_IPV6_OPEN_RE = re.compile(r"\[|%5[Bb]")
_IPV6_CLOSE_RE = re.compile(r"\]|%5[Dd]")
# Where an authority ends. A backslash ends one too for a special scheme, but
# ``safe_url`` has already refused every address still holding one.
_AUTHORITY_END_RE = re.compile(r"[/?#]")
_TITLE_MAX = 200
_LABEL_MAX = 64
@dataclass(frozen=True)
class CitationSource:
"""One citable search result, as the backend received it (untrusted)."""
ref_id: str
title: str = ""
url: str = ""
@dataclass(frozen=True)
class Citation:
"""A resolved citation, numbered in first-appearance order within a message."""
index: int
ref_id: str
title: str
url: str
label: str
def to_payload(
self, *, spans: list[list[int]], body_sha256: str
) -> dict[str, Any]:
"""The persisted sidecar shape (``message.content.citations`` entries).
``spans`` are half-open ``[start, end)`` ranges in UTF-16 code units,
and ``body_sha256`` is the digest of the exact body they were measured
in. Neither means anything without the other, so neither is optional.
"""
return {
"index": self.index,
"ref_id": self.ref_id,
"title": self.title,
"url": self.url,
"label": self.label,
"spans": [list(span) for span in spans],
"body_sha256": body_sha256,
}
@dataclass(frozen=True)
class WrittenLink:
"""One link a registered marker writes: a citation, or the fallback label.
``plain`` is the same attribution outside Markdown. Quick-reply labels and
file labels are lifted out of the body into structured fields that no
surface parses as Markdown, so writing a link there would show the reader
its syntax; the domain on its own still says who is being cited.
"""
index: Optional[int]
text: str
plain: str
@dataclass(frozen=True)
class RegisteredMarker:
"""One marker registered at the native boundary, under an opaque token."""
token: str
# The marker exactly as it arrived, restored when the token has ended up
# somewhere a link must not be written.
literal: str
lead: str
parts: tuple[WrittenLink, ...]
@dataclass(frozen=True)
class CitationBundle:
"""Everything one message's delivery needs to write its citations.
Immutable all the way down, and the ``sources`` are a snapshot: the map a
backend hands to registration is live thread state that keeps being written
to, and a bundle that read it later would describe a different message.
"""
citations: tuple[Citation, ...]
markers: tuple[RegisteredMarker, ...]
sources: tuple[CitationSource, ...]
def clean_title(value: Any) -> str:
"""Collapse an untrusted source title to one safe single-line string."""
if not isinstance(value, str):
return ""
return " ".join(_CONTROL_RE.sub(" ", value).split())[:_TITLE_MAX].strip()
def _is_replaced_code_point(code: int) -> bool:
"""Whether the renderer substitutes U+FFFD for the numeric reference *code*.
This is micromark's table (``micromark-util-decode-numeric-character-
reference``), because micromark is the parser behind the Web renderer and
the only table that can keep a persisted URL equal to the href drawn from
it. It is wider than CommonMark's own wording, which replaces just NUL,
surrogates and out-of-range values: micromark also replaces the C0 controls
a URL may not hold, the whole C1 range, and the noncharacters. Notably it
does *not* apply HTML's Windows-1252 mapping, so ``€`` becomes U+FFFD
rather than ``€`` - resolving it to ``€`` would name a page the rendered
link never opens.
"""
return (
code < 9
or code == 11
or 13 < code < 32
or 126 < code < 160
or 55295 < code < 57344
or 64975 < code < 65008
or code & 0xFFFF in (0xFFFE, 0xFFFF)
or code > 0x10FFFF
)
def _resolve_references(value: str) -> str:
"""Resolve one round of CommonMark character references."""
if "&" not in value:
return value
def replace(match: re.Match[str]) -> str:
body = match.group(1)
if body[0] == "#":
try:
code = int(body[2:], 16) if body[1] in "xX" else int(body[1:])
except ValueError:
return match.group(0)
return "\ufffd" if _is_replaced_code_point(code) else chr(code)
return HTML5_ENTITIES.get(f"{body};", match.group(0))
return _REFERENCE_RE.sub(replace, value)
def _normalize_destination(value: str) -> str:
"""The provider's URL string as a browser reads it, before Markdown escaping.
The two passes are ordered the way the pipeline applies them, because they
disagree about the same character. A *literal* tab or newline is cleaned off
first, the way WHATWG's URL parser cleans its own input - removed wherever it
sits, with leading and trailing C0 controls and spaces trimmed - and it has
to be: a literal tab in a Markdown destination ends the destination, so
``[x](https://example.com/p<TAB>q)`` renders as no link at all.
A tab a *character reference* spells is a different character. Providers lift
URLs out of HTML, so one round of references is resolved after that cleanup,
and what it produces is part of the destination: micromark percent-encodes
it, so ``p	q``, ``p	q`` and ``p
q`` have the hrefs ``p%09q``,
``p%09q`` and ``p%0Aq``. Resolving before the cleanup deleted exactly those,
pointing the citation at ``p	q`` rendered as ``pq`` - a different page,
silently.
Everything else a reference produces survives to be percent-encoded too,
because that is what the browser does with it. Deleting it instead - which a
blanket control scrub either side of the reference pass used to do - moved
the page the same way: ``https://example.com/p€q`` became
``https://example.com/pq`` rather than the ``p%EF%BF%BDq`` the renderer
produces.
"""
return _resolve_references(_URL_REMOVED_RE.sub("", value).strip(_URL_TRIMMED))
def _ipv4_number(part: str) -> Optional[int]:
"""One dotted part as WHATWG's IPv4 number parser reads it, or ``None``.
A leading ``0x`` makes the part hexadecimal and a bare leading ``0`` makes
it octal, which is how ``0x7f.1`` and ``017700000001`` both reach
``127.0.0.1``.
A part written wider than 32 bits is still a number - a browser reads it and
then fails the address - so it answers with a value past the cap rather than
with ``None``, which would call it a domain name instead. That also keeps
``int`` off a host spelled with thousands of digits, which CPython refuses to
convert at all (``sys.set_int_max_str_digits``): an unhandled ``ValueError``
there would have escaped ``safe_url`` on nothing but an untrusted string.
"""
if not part:
return None
digits, radix = _DECIMAL_DIGITS, 10
if len(part) > 1 and part[0] == "0":
if part[1] in "xX":
part, radix, digits = part[2:], 16, _HEX_DIGITS
if not part:
return 0
else:
part, radix, digits = part[1:], 8, _OCTAL_DIGITS
if not all(char in digits for char in part):
return None
if len(part.lstrip("0")) > _IPV4_WIDEST[radix]:
return _IPV4_TOO_LARGE
return int(part, radix)
def _canonical_host(host: str) -> str:
"""*host* as a browser serializes it, or ``""`` when it names no host.
WHATWG hands any host whose last label is a number to its IPv4 parser, so
``2130706433``, ``0x7f.1``, ``127.1`` and ``017700000001`` all name
``127.0.0.1``. An untrusted search result that spells a loopback or
private-network address that way would otherwise be attributed to the digits
themselves, hiding where the link goes; and a form the parser refuses
(``256.1.1.1``, ``1.2.3.4.5``, ``example.com.0x1``) names no host at all.
"""
parts = host.split(".")
if parts[-1] == "" and len(parts) > 1:
# A trailing dot is not a label. A browser keeps it on a domain
# (``example.com.``) but ignores it when reading a number.
parts = parts[:-1]
last = parts[-1]
ends_in_number = bool(last) and all(char in _DECIMAL_DIGITS for char in last)
if not ends_in_number and _ipv4_number(last) is None:
return host
if len(parts) > 4:
return ""
numbers: list[int] = []
for part in parts:
number = _ipv4_number(part)
if number is None:
return ""
numbers.append(number)
if any(number > 255 for number in numbers[:-1]):
return ""
if numbers[-1] >= 256 ** (5 - len(numbers)):
return ""
address = numbers[-1]
for offset, number in enumerate(numbers[:-1]):
address += number * 256 ** (3 - offset)
return ".".join(str((address >> shift) & 0xFF) for shift in (24, 16, 8, 0))
def _canonical_uri(value: str) -> str:
"""The destination form that survives the Markdown pipeline unchanged.
``remark`` resolves a destination's references and escapes, then percent-
encodes what is left. Both are applied here so the URL the backend persists
is byte-identical to the href the renderer produces from it - which is the
only reason a badge can be matched to its own link.
The percent-encoding is ``spell_uri``, the one rule the renderer applies
and every consumer of a resolved destination needs. Only what a citation
needs on top of it is written here, and it is written as escapes that the
shared rule then carries through unchanged.
"""
return spell_uri(
_CITATION_UNSAFE_RE.sub(lambda match: percent_encode(match.group()), value)
)
def _parse_ipv6(value: str) -> Optional[list[int]]:
"""WHATWG's IPv6 parser: eight 16-bit pieces, or ``None`` for no address.
Python's own ``ipaddress`` cannot stand in here. It accepts a zone id
(``fe80::1%eth0``) that a browser refuses, and it serializes an
IPv4-mapped address back as ``::ffff:127.0.0.1`` where a browser shows
``::ffff:7f00:1`` - so a label drawn from it would name the address bar
shows something the reader never sees.
"""
address = [0] * 8
piece_index = 0
compress: Optional[int] = None
pointer = 0
length = len(value)
def char(offset: int = 0) -> str:
position = pointer + offset
return value[position] if position < length else ""
if char() == ":":
if char(1) != ":":
return None
pointer += 2
piece_index += 1
compress = piece_index
while pointer < length:
if piece_index == 8:
return None
if char() == ":":
if compress is not None:
return None
pointer += 1
piece_index += 1
compress = piece_index
continue
piece_value = 0
piece_length = 0
while piece_length < 4 and char() in _HEX_DIGITS:
piece_value = piece_value * 0x10 + int(char(), 16)
pointer += 1
piece_length += 1
if char() == ".":
# A dotted tail, as in ``::ffff:127.0.0.1``. It is read as an IPv4
# address and stored in the last two pieces, which is why the
# browser serializes it back as hexadecimal.
if piece_length == 0 or piece_index > 6:
return None
pointer -= piece_length
numbers_seen = 0
while pointer < length:
ipv4_piece: Optional[int] = None
if numbers_seen > 0:
if char() == "." and numbers_seen < 4:
pointer += 1
else:
return None
if char() not in _DECIMAL_DIGITS:
return None
while char() in _DECIMAL_DIGITS:
number = int(char())
if ipv4_piece is None:
ipv4_piece = number
elif ipv4_piece == 0:
return None
else:
ipv4_piece = ipv4_piece * 10 + number
if ipv4_piece > 255:
return None
pointer += 1
address[piece_index] = address[piece_index] * 0x100 + (ipv4_piece or 0)
numbers_seen += 1
if numbers_seen in (2, 4):
piece_index += 1
if numbers_seen != 4:
return None
break
if char() == ":":
pointer += 1
if pointer >= length:
return None
elif char():
return None
address[piece_index] = piece_value
piece_index += 1
if compress is not None:
swaps = piece_index - compress
piece_index = 7
while piece_index != 0 and swaps > 0:
address[piece_index], address[compress + swaps - 1] = (
address[compress + swaps - 1],
address[piece_index],
)
piece_index -= 1
swaps -= 1
elif piece_index != 8:
return None
return address
def _serialize_ipv6(address: list[int]) -> str:
"""The address as a browser writes it: lowercase, longest zero run elided."""
compress: Optional[int] = None
best_length = 1
run_start: Optional[int] = None
run_length = 0
for piece_index in range(8):
if address[piece_index] != 0:
run_start = None
run_length = 0
continue
if run_start is None:
run_start = piece_index
run_length += 1
# Strictly greater keeps the leftmost of two equally long runs.
if run_length > best_length:
best_length = run_length
compress = run_start
out: list[str] = []
ignore_zero = False
for piece_index in range(8):
if ignore_zero and address[piece_index] == 0:
continue
ignore_zero = False
if compress == piece_index:
out.append("::" if piece_index == 0 else ":")
ignore_zero = True
continue
out.append(format(address[piece_index], "x"))
if piece_index != 7:
out.append(":")
return "".join(out)
_ACE_PREFIX = "xn--"
def _ace_label(label: str) -> str:
"""*label*, an ``xn--`` spelling, when it round-trips; ``""`` otherwise."""
try:
decoded = label[len(_ACE_PREFIX) :].encode("ascii").decode("punycode")
except (UnicodeError, ValueError):
return ""
if not decoded or decoded.isascii():
return ""
try:
encoded = idna.encode(
decoded, uts46=True, transitional=False, std3_rules=False
).decode("ascii")
except (UnicodeError, ValueError):
return ""
return encoded if encoded == label else ""
def _canonical_domain(host: str) -> str:
"""A non-IP host as a browser serializes it, or ``""`` when it names none.
An internationalized label is canonicalized the way a browser does it:
non-transitional UTS #46, with the STD3 and hyphen rules off. Python's own
``"idna"`` codec is IDNA 2003 *transitional* instead, which folds ``ß`` into
``ss`` - so ``https://faß.de/`` would be attributed to ``fass.de`` while the
link opens ``xn--fa-hia.de``, naming a domain the reader never visits.
"""
if not host:
return ""
try:
decoded = unquote(host, errors="strict")
except UnicodeDecodeError:
return ""
if any(char in _FORBIDDEN_DOMAIN for char in decoded):
return ""
labels: list[str] = []
for label in decoded.split("."):
if label.isascii() and not label.lower().startswith(_ACE_PREFIX):
# A browser passes an ASCII label through untouched apart from case,
# even one UTS #46 refuses: ``my_site.example.com`` holds an
# underscore and ``ab--cd.example`` trips the hyphen rule, and both
# resolve. Running them through IDNA would lose the citation.
labels.append(label.lower())
continue
if label.isascii():
# An ``xn--`` label is the one ASCII spelling a browser does not
# pass through: it decodes the punycode and refuses the whole URL
# when that fails (``https://xn--/x`` does not open). Accept one
# only when it is the exact encoding of a label the path below
# accepts, so a citation never names a host its link cannot reach.
ace = _ace_label(label.lower())
if not ace:
return ""
labels.append(ace)
continue
try:
# A browser labels an internationalized host by the punycode it
# actually visits, so a citation has to attribute that and not the
# bytes the URL spells.
labels.append(
idna.encode(
label, uts46=True, transitional=False, std3_rules=False
).decode("ascii")
)
except (UnicodeError, ValueError):
# Stricter than a browser on the margins: IDNA 2008 disallows symbol
# labels UTS #46 alone would map (``❤.example`` resolves for a
# browser, and is dropped here). Rejection degrades the citation to
# its unresolved label, which is the only safe direction - a label
# that names a different domain than the link opens is the defect.
return ""
return _canonical_host(".".join(labels))
def _port_accepted(value: str) -> bool:
"""Whether a browser would read *value* as a port at all.
``str.isdigit`` is true for ``٣`` and ``int`` reads that as 3, so a port
checked with it would let ``:٣`` through as 3 while a browser refuses the
URL outright. Membership in the ASCII digits is the only test that agrees.
An empty port is not a rejection - ``https://example.com:/x`` opens - and a
port in range is kept as the provider wrote it, default or not: it is the
same page either way, and the label never shows a port.
"""
if not value:
return True
if not all(char in _DECIMAL_DIGITS for char in value):
return False
trimmed = value.lstrip("0")
# Bound the string before converting it: CPython refuses to convert a
# decimal ``int`` past ``sys.set_int_max_str_digits``, and an untrusted
# authority is exactly where a host spelled with thousands of digits shows
# up. Anything wider than the maximum port is out of range regardless.
if len(trimmed) > _PORT_DIGITS:
return False
return not trimmed or int(trimmed) <= _PORT_MAX
def _authority(
url: str, scheme: str, *, literal_brackets: bool
) -> Optional[tuple[str, str, str]]:
"""``(authority, label host, rest)`` for *url*, or ``None`` to reject it.
*literal_brackets* says whether the destination this *url* was spelled from
wrapped its host in LITERAL brackets. It is a required argument because the
spelled URL no longer holds the answer - ``[`` and ``]`` are outside the
URI-safe set, so spelling writes both ``[::1]`` and an already-escaped
``%5B::1%5D`` as the same ``%5B::1%5D`` - and guessing it from the spelling
is how a destination that named no host at all acquired a live one. Ask
``_authority_brackets_literal`` of the resolved destination, once, before it
is spelled.
One function answers the whole authority, because the parts are not
independent: which ``@`` starts the host depends on the userinfo, whether a
``:`` starts a port depends on whether the host is a bracketed IPv6
literal, and whether a port is in range depends on nothing else at all.
Asking those questions separately - which is what reading
``urlsplit().hostname`` and nothing else amounted to - answers each one
against a different authority, and every shape that fell between two of
them became a citation pointing somewhere the reader could not go.
It answers two things at once because they are one judgment. Whether a
browser can open this URL decides if the citation is delivered; what that
browser resolves the host to decides what the label may claim. The
*authority* it hands back is still the one the provider wrote: a page is
named by the URL its source gave, and a host written ``2130706433``,
``EXAMPLE.com`` or ``:0080`` opens the same page the label names.
The canonical URI is what gets parsed, not the provider's raw string, so
this reads the same authority the renderer will hand the browser.
"""
remainder = url[len(scheme) + 1 :]
if not remainder.startswith("//"):
return None
remainder = remainder[2:]
end = _AUTHORITY_END_RE.search(remainder)
authority = remainder[: end.start()] if end else remainder
rest = remainder[end.start() :] if end else ""
# WHATWG splits userinfo at the LAST ``@``: everything before it is
# credentials, which may hold an ``@`` of their own. They are carried
# across untouched - case-folding a password would change it.
userinfo, separator, host_port = authority.rpartition("@")
if not separator:
userinfo, host_port = "", authority
opener = _IPV6_OPEN_RE.match(host_port)
if opener is not None:
if not literal_brackets:
# The wrapper is here in the spelling and was not in the
# destination: the provider wrote its own ``%5B``, which is not
# authority syntax but two characters a host may not hold. Reading
# it as a bracket would mint an address the destination never named.
return None
closer = _IPV6_CLOSE_RE.search(host_port, opener.end())
if closer is None:
return None
inner = host_port[opener.end() : closer.start()]
address = _parse_ipv6(inner)
if address is None:
return None
# Literal brackets, not the ``%5B`` the URI encoder left in their
# place: this host WAS written in brackets, and no URL parser opens the
# escaped spelling, so the badge could not reach the address either.
host = f"[{inner}]"
label_host = f"[{_serialize_ipv6(address)}]"
port = host_port[closer.end() :]
if port and not port.startswith(":"):
return None
else:
host, colon, port_text = host_port.partition(":")
label_host = _canonical_domain(host)
if not label_host:
return None
port = f":{port_text}" if colon else ""
if not _port_accepted(port[1:]):
return None
authority = f"{host}{port}"
return (f"{userinfo}@{authority}" if userinfo else authority), label_host, rest
def _authority_brackets_literal(resolved: str) -> bool:
"""Whether *resolved* wraps its host in literal brackets, not escapes.
*resolved* is the destination as a Markdown reader resolved it and BEFORE
any URI spelling: the one moment the two spellings are still distinct. A
provider that wrote ``https://[::1]/admin`` named a host; a provider that
wrote ``https://%5B::1%5D/admin`` named two characters a host may not hold,
and the difference survives nowhere else - spelling writes both the same way.
Both ends of the wrapper have to be literal. A mixed one (``[::1%5D``,
``%5B::1]``) is not a bracketed host with an escape in it; it is a host that
was never written in brackets at all, and promoting half of it into syntax
invents an address out of the other half.
The authority is read here the way ``_authority`` reads it there, and the
two agree because spelling cannot move the boundaries: ``/``, ``?``, ``#``,
``@`` and ``:`` are all inside the URI-safe set, so they are the same
characters in the same places before and after. What a character reference
or a backslash escape resolved to counts as written - that resolution
already happened, once, in ``_normalize_destination``.
"""
scheme, separator, remainder = resolved.partition(":")
if not separator or not remainder.startswith("//"):
return False
remainder = remainder[2:]
end = _AUTHORITY_END_RE.search(remainder)
authority = remainder[: end.start()] if end else remainder
_, separator, host_port = authority.rpartition("@")
if not separator:
host_port = authority
opener = _IPV6_OPEN_RE.match(host_port)
if opener is None or opener.group() != "[":
return False
closer = _IPV6_CLOSE_RE.search(host_port, opener.end())
return closer is not None and closer.group() == "]"
def spell_destination(resolved: str) -> str:
"""Spell a resolved destination, keeping a literal IPv6 host readable.
Takes the destination BEFORE spelling and returns it after, because the two
halves of this job cannot be separated: ``[`` and ``]`` are two of the
characters ``spell_uri`` escapes, so ``https://[::1]/x`` leaves it spelled
``https://%5B::1%5D/x``, which is not an address any parser opens - and by
then nothing in the string says whether those escapes were a host's own
syntax or the provider's own characters. A caller that could hand over only
the spelled URL could only guess, and guessing turned
``https://%5B::1%5D/admin`` into a live link to the loopback interface.
The brackets go back in the authority and nowhere else: the same two
characters in a path, query, fragment or userinfo are data, and stay
spelled. They go back only when the authority they belong to parses, so a
host like ``[nope]`` stays escaped rather than becoming a different URL, and
every address without a literal bracketed host is returned exactly as
``spell_uri`` wrote it.
"""
spelled = spell_uri(resolved)
if not _authority_brackets_literal(resolved):
return spelled
scheme, separator, remainder = spelled.partition(":")
if not separator or not remainder.startswith("//"):
return spelled
parsed = _authority(spelled, scheme, literal_brackets=True)
if parsed is None:
return spelled
return f"{scheme}://{parsed[0]}{parsed[2]}"
def safe_url(value: Any) -> str:
"""Canonical http(s) URL usable as a Markdown destination, or ``""``.
Anything else - ``javascript:``, ``data:``, ``file:``, a hostless URL, an
authority no browser would open - is rejected rather than repaired, so the
citation is reported as unresolved instead of becoming an unsafe or dead
link.
What survives is left as it arrived, percent-encoding aside: the URL names
the page its source gave, and this is not the place to decide that two
spellings of one are the same. ``source_label`` is where the host a browser
resolves is worked out, because that is the one place the difference is
something the reader is shown.
"""
if not isinstance(value, str):
return ""
raw = _normalize_destination(value)
if not raw:
return ""
# A browser reads a backslash in an http(s) authority or path as ``/`` and
# one in a query as data, so there is no single spelling of it that names
# the page for every reader: ``https://trusted.example\\@attacker.example/``
# opens trusted.example and, percent-encoded, is attributed to
# attacker.example. The citation is refused rather than guessed at. An
# already-encoded ``%5C`` is data to every reader and is kept.
if "\\" in raw:
return ""
url = _canonical_uri(raw)
scheme, separator, _ = url.partition(":")
# A browser lowercases the scheme, and a renderer that only knows the
# lowercase spelling does not see a link at all: Telegram delivered
# ``[example.com](HTTPS://Example.com/X)`` as raw Markdown.
scheme = scheme.lower()
if not separator or scheme not in _ALLOWED_SCHEMES:
return ""
# Qualified from ``raw``: the resolved destination, before ``_canonical_uri``
# spelled it. After spelling there is no such thing as a bracket to ask about.
parsed = _authority(
url, scheme, literal_brackets=_authority_brackets_literal(raw)
)
if parsed is None:
return ""
return f"{scheme}://{parsed[0]}{parsed[2]}"
def source_label(url: str, title: str = "") -> str:
"""Short link text for a citation: its domain, which is what attributes it."""
host = ""
if isinstance(url, str):
scheme, separator, _ = url.partition(":")
if separator:
# A label is worked out from a URL ``safe_url`` already accepted,
# which is spelled with its brackets literal if it has any.
parsed = _authority(
url,
scheme.lower(),
literal_brackets=_authority_brackets_literal(url),
)
if parsed is not None:
# The port is left out on purpose. A page is attributed by the
# site it is on, a non-default port does not change which site
# that is, and showing it would spend six of the label's 64
# characters saying so.
host = parsed[1]
if host.startswith("www."):
host = host[4:]
if not host:
return title[:_LABEL_MAX].strip()
if len(host) <= _LABEL_MAX:
return host
# Shorten a long host from the LEFT, keeping the tail verbatim: the label is
# always either the host or a suffix of it, marked by the ellipsis.
#
# A label cut from the right names a site the link never opens -
# ``developers.openai.com.<padding>.attacker.example`` shown as
# ``developers.openai.com…`` - and keeping only whole labels degrades the
# other way: ``<62 chars>.co.uk`` would be shown as ``…co.uk``, a public
# suffix every site under it shares and the perfect place to hide a long
# attacker label. So the budget is filled from the right instead, and only
# the leftmost piece - the one the ellipsis is attached to - may be partial.
#
# That cannot compress a host into a different plausible domain, because the
# two cases exclude each other. A short fragment only happens when whole
# labels already fill most of the budget, and those are the rightmost labels,
# so the registrable domain is shown whole and the fragment sits in a
# subdomain slot. A fragment of the registrable label itself only happens
# when little else fits, which leaves it most of the 63 characters - more
# than a DNS label may hold minus a few, so nothing recognizable can hide in
# what is left out.
return "…" + host[-(_LABEL_MAX - 1) :]
# Every character CommonMark (plus GFM) still reads as syntax inside link
# text. Two different kinds of damage, one answer: a backtick, an angle bracket
# or an unbalanced square bracket opens a construct that runs past ``](url)``
# and swallows the link itself, while emphasis, strikethrough, a character
# reference or a table cell divider quietly change which characters the reader
# is shown. The label is the source's domain - the whole attribution claim - so
# a label that reads as ``ab.example`` when the link goes to ``a*b*.example``
# is the same defect as no link at all.
_LABEL_SYNTAX_RE = re.compile(r"([\\\[\]`<*_~&|])")
def _escape_label(value: str) -> str:
"""Escape every character a Markdown reader would not show verbatim.
What comes out is a label that reads back exactly as the sidecar spells it
on every surface. The Web renderer resolves the escapes itself; each IM
formatter resolves them once, ahead of its own dialect pass, through
``modules.im.formatters.hold_markdown_escapes`` - so no reader is shown a
backslash that was only ever there to keep a domain intact.
"""
return _LABEL_SYNTAX_RE.sub(r"\\\1", value)
def has_citation_markers(text: Optional[str]) -> bool:
"""True when ``text`` carries at least one complete citation marker."""
return bool(text) and _START in text and bool(CITATION_MARKER_RE.search(text))
def citation_ref_ids(text: Optional[str]) -> list[str]:
"""The refs a message asks to have attributed, in first-appearance order.
Only markers a citation can actually be written for are requests: one
inside a code example is literal text, one inside a hidden block leaves
with the block, and one written into an address, a title, or a reference
identifier has nowhere a link could be shown. This is the same question
registration asks, asked of the same mask - a marker that will not be
attributed must not make delivery wait for a source either, or hydrate one
into a sidecar nothing points at. A malformed ref is not a request either -
it can name no source, so the marker degrades to the unresolved label no
matter what arrives later.
"""
if not text or _START not in text:
return []
requested: dict[str, None] = {}
mask = mask_citation_slots(text, transparent=_SLOT_TRANSPARENT_RE)
for match in CITATION_MARKER_RE.finditer(mask):
for ref in match.group(1).split(_SEP):
if ref and _REF_ID_RE.fullmatch(ref):
requested.setdefault(ref, None)
return list(requested)
def unresolved_refs(
ref_ids: Iterable[str], sources: Mapping[str, CitationSource]
) -> list[str]:
"""Which of *ref_ids* no source has been recorded for yet, in the given order.
This is the question a delivery boundary asks: could a search still arrive
that changes how this message reads? A ref whose source is already present
has reached its final form even when that source turns out to be unlinkable -
waiting on it would hold the message forever - and a malformed ref can never
name a source at all, so neither is reported here.
"""
return [ref for ref in ref_ids if _REF_ID_RE.fullmatch(ref) and ref not in sources]
def register_citations(
text: Optional[str],
sources: Mapping[str, CitationSource],
*,
unresolved_label: str,
) -> tuple[Optional[str], Optional[CitationBundle]]:
"""Give every marker this message will attribute a private identity, now.
``sources`` maps ref_id to the search result the backend captured for it,
scoped to the native thread the message belongs to. Returns the text to
carry through delivery and the bundle that finalizes it; sources repeated
across markers share one entry (and one index) keyed by their URL.
Registration happens against the text exactly as the backend produced it,
because that is the only version of it whose citation grammar means what it
says. Every later stage rewrites the text - a ``<silent>`` block leaves, a
``file://`` link is stripped to its label, images are appended - and a
rewrite can splice two ordinary halves into something that reads as a
marker. ``\\ue200ci<silent>x</silent>te\\ue202turn0view0\\ue201`` is one
marker and one accidental one to a reader of the delivered text, and a
stage that asked the delivered text which markers to attribute would put a
citation on text the model never cited. So the question is asked once, here,
and what travels from here on is an opaque token per registered marker:
unguessable, never derived from the ref or from its order, and checked
against the arriving text so a token can only be a token this call minted.
A marker inside a code example, a ``<silent>`` block, or a Markdown slot
that holds data rather than prose is not registered (one mask decides all
three); one whose refs all fail to resolve with no label to fall back to
keeps its literal text, exactly as before.
"""
if not text or _START not in text:
return text, None
fallback = (unresolved_label or "").strip()
citations: list[Citation] = []
by_url: dict[str, Citation] = {}
consulted: dict[str, CitationSource] = {}
markers: list[RegisteredMarker] = []
minted: set[str] = set()
def mint_token() -> str:
# A nonce, not a name: the token is the citation's identity for the rest
# of delivery, so nothing about it may be reconstructible from the
# message. Colliding with text that already arrived would hand that text
# the identity, so the arriving text is checked too.
while True:
token = f"{_TOKEN_OPEN}{secrets.token_hex(_TOKEN_NONCE_BYTES)}{_TOKEN_CLOSE}"
if token not in minted and token not in text:
minted.add(token)
return token
def resolve_ref(ref: str) -> Optional[Citation]:
if not _REF_ID_RE.fullmatch(ref):
return None
source = sources.get(ref)
if source is None:
return None
consulted.setdefault(ref, source)
url = safe_url(source.url)
if not url:
return None
existing = by_url.get(url)
if existing is not None:
return existing
title = clean_title(source.title)
label = source_label(url, title)
if not label:
return None
citation = Citation(
index=len(citations) + 1,
ref_id=ref,
title=title,
url=url,
label=label,
)
citations.append(citation)
by_url[url] = citation
return citation
def replace(match: re.Match[str]) -> str:
refs = [ref for ref in match.group(1).split(_SEP) if ref]
parts: list[WrittenLink] = []
unresolved = not refs
# Markers usually sit flush against the preceding word or full stop;
# a separating space keeps the link from reading as part of the sentence.
lead = "" if match.start() == 0 or text[match.start() - 1].isspace() else " "
for ref in refs:
citation = resolve_ref(ref)
if citation is None:
unresolved = True
continue
parts.append(
WrittenLink(
index=citation.index,
text=_link_spelling(citation),
plain=citation.label,
)
)
if unresolved and fallback:
parts.append(WrittenLink(index=None, text=fallback, plain=fallback))
if not parts:
return match.group(0)
token = mint_token()
markers.append(
RegisteredMarker(
token=token,
literal=match.group(0),
lead=lead,
parts=tuple(parts),
)
)
return token
registered = _rewrite_citable_markers(text, replace)
if not markers:
return registered, None
return registered, CitationBundle(
citations=tuple(citations),
markers=tuple(markers),
sources=tuple(consulted.values()),
)
class TextCitation(Protocol):
"""A citation a provider attached to a text block (C-1 ``Citation``)."""
url: str
title: Optional[str]
start: Optional[int]
end: Optional[int]
def register_text_citations(
blocks: Iterable[tuple[str, str, Iterable[TextCitation]]],
*,
unresolved_label: str,
transform: Callable[[str], str] = lambda text: text,
) -> tuple[str, Optional[CitationBundle]]:
"""Register the citations providers attach to text blocks, as ``register_citations`` does markers.
The one builder from ``(separator, text, TextBlock.citations)`` to the delivered
shape (``vibey-hosted-tools.md`` §4.2): each block's text after its separator,
what goes before it in the display (nothing between adjacent blocks, which are
one span of prose), with an opaque token where each citation is written out at
delivery, and the bundle that writes Markdown links plus the Web's badge sidecar.
A citation with a range is attributed where its cited span ends (OpenAI and
xAI ``url_citation``); one without covers its whole block, so it is
attributed at the block's end (Anthropic). ``transform`` runs on the joined
text before registration - the display rules, such as dropping silent
blocks - and a citation whose place it removed, or that falls in code, is
left out rather than written as a stray marker.
"""
blocks = list(blocks)
# Each ref this call inserts carries a nonce the text does not contain, so no marker the text itself
# carries (a literal one in a code example, say) can name an annotation's ref, at registration or at
# cleanup.
arrived = "".join(separator + text for separator, text, _ in blocks)
nonce = secrets.token_hex(_TOKEN_NONCE_BYTES)
while nonce in arrived:
nonce = secrets.token_hex(_TOKEN_NONCE_BYTES)
sources: dict[str, CitationSource] = {}
joined = ""
for block_index, (separator, text, citations) in enumerate(blocks):
at: dict[int, list[str]] = {}
# A block the display rules remove entirely shows nothing to cite; its
# end-of-block citation would otherwise survive past the removed text.
if not transform(text).strip():
citations = ()
for citation_index, citation in enumerate(citations or ()):
ref = f"{nonce}:t{block_index}c{citation_index}"
sources[ref] = CitationSource(ref_id=ref, title=citation.title or "", url=citation.url)
end = len(text) if citation.end is None else min(max(citation.end, 0), len(text))
at.setdefault(end, []).append(ref)
marked = text
for offset in sorted(at, reverse=True):
marker = f"{_START}cite{_SEP}{_SEP.join(at[offset])}{_END}"
marked = marked[:offset] + marker + marked[offset:]
if marked:
joined += (separator if joined else "") + marked
joined = transform(joined)
if not sources:
return joined, None
registered, bundle = register_citations(joined, sources, unresolved_label=unresolved_label)
# Only the markers inserted here for annotations whose place was left out (in code, or removed by the
# display rules) go: only they name this call's refs. A marker the text itself carries, such as a literal
# one in a code example that registration deliberately left alone, stays as written.
def drop_synthetic(match: re.Match[str]) -> str:
refs = [ref for ref in match.group(1).split(_SEP) if ref]
return "" if refs and all(ref in sources for ref in refs) else match.group(0)
return CITATION_MARKER_RE.sub(drop_synthetic, registered or ""), bundle
def materialize_citations(
text: Optional[str],
bundle: Optional[CitationBundle],
*,
as_markdown: bool = True,
) -> Optional[str]:
"""Write the registered citations into *text*, with no sidecar.
This is the form every surface that carries only text needs - an IM copy, a
Turn snapshot, an agent-run record, a live stream chunk. ``as_markdown``
off writes each citation as its bare domain instead of a Markdown link, for
the structured fields (quick-reply labels, file labels) that are extracted
out of the body and are not Markdown anywhere.
"""
if not text or bundle is None or _TOKEN_OPEN not in text:
return text
written, _ = _write_citations(text, bundle, as_markdown=as_markdown)
return written
def finalize_citations(
text: Optional[str],
bundle: Optional[CitationBundle],
) -> tuple[Optional[str], list[dict[str, Any]]]:
"""Write the registered citations into *text* and bind the sidecar to it.
The returned rows describe THIS body and no other. Each carries the exact
``spans`` its links occupy in it and the digest of the body itself, because
a span alone is not an identity: two ordinary links can be spelled the same,
and a paragraph removed above one moves another into its place. Rendering a
badge from a span the producer measured in a different version of the text
is how an ordinary link gets to impersonate a cited one, so a consumer
checks the digest first and the spans only after it matches.
Only tokens this bundle minted are written; anything else in the text is
left exactly as it arrived. A token that has landed inside code or a hidden
block by the time delivery settles gets its original marker text back rather
than a link or an exposed internal token - the reader is shown what the
model wrote, and nothing claims it was cited.
"""
if not text or bundle is None or _TOKEN_OPEN not in text:
return text, []
body, placed = _write_citations(text, bundle, as_markdown=True)
if not placed:
return body, []
offsets = _utf16_offsets(
body, (offset for _, start, end in placed for offset in (start, end))
)
spans: dict[int, list[list[int]]] = {}
for index, start, end in placed:
spans.setdefault(index, []).append([offsets[start], offsets[end]])
digest = body_digest(body)
return body, [
citation.to_payload(spans=spans[citation.index], body_sha256=digest)
for citation in bundle.citations
if citation.index in spans
]
def resolve_citations(
text: Optional[str],
sources: Mapping[str, CitationSource],
*,
unresolved_label: str,
) -> tuple[Optional[str], list[dict[str, Any]]]:
"""Register and finalize in one step, for a surface that transforms nothing.
The two halves exist because delivery rewrites the text between them. When
nothing happens in between - a probe, a test, a caller that hands its own
output straight out - this is the same call it always was.
"""
registered, bundle = register_citations(
text, sources, unresolved_label=unresolved_label
)
return finalize_citations(registered, bundle)
def body_digest(text: str) -> str:
"""The digest a sidecar binds its spans to: SHA-256 over UTF-16 code units.
UTF-16 because that is the unit the spans are counted in, and a digest over
a different encoding of the same text would let the two disagree about which
text they mean. Little-endian, no BOM, unpaired surrogates passed through:
exactly the bytes a consumer reading the string a code unit at a time hashes.
This says which VERSION of the body a span was measured in. It is not a
signature: both sides compute it locally from text they already hold, and it
claims nothing about where that text came from.
"""
return hashlib.sha256(text.encode("utf-16-le", "surrogatepass")).hexdigest()
def _link_spelling(citation: Citation) -> str:
"""The exact Markdown link a citation is written as.
A parenthesis stays in the URL and is escaped in the destination, which
every reader resolves back to the character: an unbalanced one would
otherwise end the link early. ``safe_url`` has refused any backslash, so an
escape here cannot pair with one already in the address. An angle-bracket
destination would say the same thing, but Telegram and WeChat do not read
that form as a link.
"""
destination = _DESTINATION_PAREN_RE.sub(lambda match: "\\" + match.group(), citation.url)
return f"[{_escape_label(citation.label)}]({destination})"
def _utf16_offsets(text: str, offsets: Iterable[int]) -> dict[int, int]:
"""Convert code-point offsets into UTF-16 code-unit offsets, in one pass.
Python counts code points and every consumer of the sidecar counts UTF-16
code units, so one emoji outside the BMP puts the two a unit apart for the
rest of the message.
"""
converted: dict[int, int] = {}
cursor = 0
units = 0
for offset in sorted(set(offsets)):
units += len(text[cursor:offset].encode("utf-16-le", "surrogatepass")) // 2
cursor = offset
converted[offset] = units
return converted
def _enclosing_unit_end(
units: list[MarkdownUnit], start: int, end: int
) -> Optional[int]:
"""Where a citation for a marker inside a link or image has to go instead.
A link nested inside a link label is not a link on any surface: CommonMark
refuses to parse the inner one, and every IM dialect downstream spells the
outer one as a single wrapper. So a citation for a marker written in a
label belongs immediately after the whole unit, and after the *outermost*
one when units nest - an image inside a link is two units, and only the
link has an end a sibling can follow. ``units`` is ordered outermost-first
at each offset, so the first one that contains the marker is that unit.
"""
for unit in units:
if unit.start > start:
break
if end <= unit.end:
return unit.end
return None
def _separating_lead(parts: list[str]) -> str:
"""A space before a relocated citation, unless one is already written."""
for part in reversed(parts):
if part:
return "" if part[-1].isspace() else " "
return ""
def _write_citations(
text: str,
bundle: CitationBundle,
*,
as_markdown: bool,
) -> tuple[str, list[tuple[int, int, int]]]:
"""Replace this bundle's tokens, reporting where each citation landed.
The placements are ``(citation index, start, end)`` in code points over the
returned text, one per link actually written.
Where a citation can go is a question about the text as it stands now, so
it is asked of the final text rather than remembered from registration: a
transform may have wrapped a token in an example, and an internal token is
the one thing that may never be shown. A token that has landed in code, in
a hidden block, or in a slot the parser reads without showing gets its
original marker text back and claims nothing. A token inside a link or
image label is removed from that label and its citation written after the
unit, which is the nearest position the reader is actually shown.
``as_markdown`` off is a structured field - a file label, a quick-reply
button - that is not Markdown on any surface. There is no unit to be inside
and no link to nest, so the attribution stays where the marker was.
"""
registered = {marker.token: marker for marker in bundle.markers}
if as_markdown:
mask = mask_citation_slots(text, transparent=_SLOT_TRANSPARENT_RE)
units = markdown_link_units(text)
else:
mask = mask_hidden_and_code(text)
units = []
# ``(offset, rank, sequence, consumed, marker, literal)``. ``rank`` puts a
# citation relocated to an offset ahead of a token that starts there, and
# ``sequence`` keeps two citations relocated to the same offset in the order
# their markers were written. Both are integers, so sorting never has to
# compare the payload.
edits: list[tuple[int, int, int, int, Optional[RegisteredMarker], str]] = []
sequence = 0
for match in _TOKEN_RE.finditer(text):
marker = registered.get(match.group(0))
if marker is None:
# Token-shaped text this bundle did not mint. It arrived as ordinary
# characters and leaves as ordinary characters.
continue
consumed = match.end() - match.start()
if mask[match.start() : match.end()] != match.group(0):
edits.append(
(match.start(), 1, sequence, consumed, None, marker.literal)
)
sequence += 1
continue
relocated = _enclosing_unit_end(units, match.start(), match.end())
if relocated is None:
edits.append((match.start(), 1, sequence, consumed, marker, ""))
sequence += 1
continue
# The label keeps every byte that is not the token, and the citation is
# written as a sibling of the unit rather than a link inside its label.
edits.append((match.start(), 1, sequence, consumed, None, ""))
sequence += 1
edits.append((relocated, 0, sequence, 0, marker, ""))
sequence += 1
out: list[str] = []
placed: list[tuple[int, int, int]] = []
cursor = 0
written = 0
for offset, _rank, _sequence, consumed, marker, literal in sorted(edits):
prefix = text[cursor:offset]
out.append(prefix)
written += len(prefix)
if marker is None:
out.append(literal)
written += len(literal)
else:
# A relocated citation no longer sits where its marker did, so what
# separates it from the text is read off what has actually been
# written; one written in place keeps the lead registration measured.
lead = marker.lead if consumed else _separating_lead(out)
at = written + len(lead)
body = lead
for index, part in enumerate(marker.parts):
if index:
body += " "
at += 1
spelling = part.text if as_markdown else part.plain
if part.index is not None:
placed.append((part.index, at, at + len(spelling)))
body += spelling
at += len(spelling)
out.append(body)
written += len(body)
cursor = offset + consumed
out.append(text[cursor:])
return "".join(out), placed
def _rewrite_citable_markers(
text: str,
replace: Callable[[re.Match[str]], str],
) -> str:
"""Apply ``replace`` to every marker a citation could be written for.
A marker shown inside a code example must stay literal - an agent
explaining this very grammar is a case seen in real transcripts - and
"inside code" is a question only a CommonMark lexer can answer. Fence
lengths nest (a four-backtick block quoting a three-backtick one), an
unclosed fence swallows the rest of the document, a code span may run
across lines, and container indentation shifts all of it. A marker inside a
``<silent>`` block has to stay literal for a different reason: the block is
removed before delivery, so rewriting it would spend a citation index on
text nobody reads and lift a URL out of the block meant to hide it. Both
A marker written into a destination, a title, a reference identifier, or an
autolink's address has to stay literal for a third reason: there is no
reader-facing position there at all, so a link written into one would not be
shown - it would change where an existing link points. All three decisions
are delegated to the reply parser's offset-preserving mask: markers are
matched against the mask and spliced back into the original source, which
leaves every byte this function does not replace exactly as it arrived.
"""
mask = mask_citation_slots(text, transparent=_SLOT_TRANSPARENT_RE)
out: list[str] = []
cursor = 0
# A match in the mask cannot overlap a blanked region, so the marker text
# under it is the original text - only its surroundings may have been
# blanked, and those are copied from ``text``, never from the mask.
for match in CITATION_MARKER_RE.finditer(mask):
out.append(text[cursor : match.start()])
out.append(replace(match))
cursor = match.end()
out.append(text[cursor:])
return "".join(out)