Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

42.2 KB
Raw Blame
import re
import html
import json
import uuid
import hashlib
from collections.abc import Iterator
from typing import Any

from django.core.exceptions import ValidationError
from django.core.validators import URLValidator

from posthog.dataclasses import frozen

# Type aliases for TipTap editor nodes
TipTapNode = dict[str, Any]
TipTapContent = list[TipTapNode]

# ProseMirror node type used by Markdown notebooks.
# Keep in sync with `NotebookNodeType.MarkdownNotebook` in frontend/src/scenes/notebooks/types.ts.
MARKDOWN_NOTEBOOK_NODE_TYPE = "ph-markdown-notebook"
# ProseMirror node type used by NotebookNodeQuery on the frontend.
# Keep in sync with `NotebookNodeType.Query` in frontend/src/scenes/notebooks/types.ts.
QUERY_NODE_TYPE = "ph-query"
# QuerySchema kind that points at a saved insight by its short_id.
SAVED_INSIGHT_NODE_KIND = "SavedInsightNode"
MARKDOWN_QUERY_TAG = "Query"

# Keep in sync with `SHARED_NOTEBOOK_SUPPORTED_NODE_TYPES` in
# `frontend/src/scenes/notebooks/Nodes/sharedNodeSupport.tsx`.
SHARED_NOTEBOOK_SUPPORTED_NODE_TYPES: frozenset[str] = frozenset(
    {
        "ph-image",
        "ph-latex",
        "ph-embed",
        "ph-query",
    }
)

SHARED_NOTEBOOK_SUPPORTED_MARKDOWN_COMPONENT_TAGS: frozenset[str] = frozenset(
    {
        "Comment",
        "Divider",
        "Embed",
        "Image",
        "Latex",
        "Query",
    }
)

_SHARED_NOTEBOOK_MARKDOWN_COMPONENT_PROP_TYPES: dict[str, dict[str, type | tuple[type, ...]]] = {
    "Comment": {"ref": str, "replies": list, "text": str},
    "Divider": {},
    "Embed": {"height": (int, float), "src": str, "title": str, "width": (int, float)},
    "Image": {"alt": str, "height": (int, float), "src": str, "title": str, "width": (int, float)},
    "Latex": {"content": str, "editing": bool, "title": str},
    "Query": {
        "hideFilters": bool,
        "hideResults": bool,
        "showFilters": bool,
        "showResults": bool,
        "height": (int, float),
        "isDefaultFilterApplied": bool,
        "nodeId": str,
        "outputTab": str,
        "query": dict,
        "showSettings": bool,
        "title": str,
    },
}

_MARKDOWN_COMPONENT_START_REGEX = re.compile(r"^<[A-Z][A-Za-z0-9]*(\s|>|/)")
_MARKDOWN_COMPONENT_TAG_REGEX = re.compile(r"^<([A-Z][A-Za-z0-9]*)([\s\S]*?)(?:/>|>[\s\S]*</\1>)$")
_MARKDOWN_COMPONENT_PROP_NAME_REGEX = re.compile(r"^([A-Za-z_][A-Za-z0-9_-]*)")
_MARKDOWN_COMPONENT_RAW_PROP_VALUE_REGEX = re.compile(r"^([^\s/>]+)")
_MARKDOWN_COMPONENT_NUMBER_REGEX = re.compile(r"^-?\d+(\.\d+)?$")
_MARKDOWN_ESCAPED_COMPONENT_START_REGEX = re.compile(r"^\\<[A-Z][A-Za-z0-9]*(\s|>|/)")
_MARKDOWN_INLINE_ESCAPABLE_CHARACTERS = frozenset("\\`*_~[]()<>#+-.|!")
_MAX_MARKDOWN_COMPONENT_LINES = 1_000
_MAX_MARKDOWN_COMPONENT_CHARACTERS = 256 * 1024
_validate_notebook_embed_src = URLValidator(schemes=["http", "https"])


@frozen
class _MarkdownComponentScan:
    raw: str
    next_line_index: int
    found_terminator: bool


@frozen(frozen=False)
class _MarkdownComponentScanState:
    quote: str | None = None
    expression_depth: int = 0
    escape_next: bool = False
    awaiting_prop_value: bool = False
    opening_tag_closed: bool = False


def filter_notebook_content_for_sharing(content: Any) -> Any:
    """Return a copy of a notebook's ProseMirror document with unsupported widget nodes redacted.

    Any ``ph-*`` node not in :data:`SHARED_NOTEBOOK_SUPPORTED_NODE_TYPES` has its ``attrs`` and
    child ``content`` stripped. The original ``type`` is preserved so the frontend's allow-list
    check still renders ``UnsupportedNodePlaceholder`` without leaking the original attrs to
    anonymous viewers. Built-in ProseMirror nodes pass through unchanged.
    """
    if not isinstance(content, dict):
        return content

    node_type = content.get("type")
    if node_type == MARKDOWN_NOTEBOOK_NODE_TYPE:
        return _filter_markdown_notebook_content_for_sharing(content)

    if (
        isinstance(node_type, str)
        and node_type.startswith("ph-")
        and node_type not in SHARED_NOTEBOOK_SUPPORTED_NODE_TYPES
    ):
        return {"type": node_type}

    filtered: dict[str, Any] = {k: v for k, v in content.items() if k != "content"}
    children = content.get("content")
    if isinstance(children, list):
        filtered["content"] = [_filter_notebook_child_content_for_sharing(child) for child in children]
    elif "content" in content:
        filtered["content"] = children
    return filtered


def _filter_notebook_child_content_for_sharing(child: Any) -> Any:
    if isinstance(child, dict) and child.get("type") == MARKDOWN_NOTEBOOK_NODE_TYPE:
        return _filter_markdown_notebook_content_for_sharing(child)
    return filter_notebook_content_for_sharing(child)


def _filter_markdown_notebook_content_for_sharing(content: TipTapNode) -> TipTapNode:
    attrs = content.get("attrs")
    if not isinstance(attrs, dict):
        return {"type": MARKDOWN_NOTEBOOK_NODE_TYPE}

    filtered_attrs: dict[str, Any] = {}
    node_id = attrs.get("nodeId")
    if isinstance(node_id, str):
        filtered_attrs["nodeId"] = node_id

    markdown = attrs.get("markdown")
    if isinstance(markdown, str):
        filtered_attrs["markdown"] = _filter_markdown_components_for_sharing(markdown)

    return {"type": MARKDOWN_NOTEBOOK_NODE_TYPE, "attrs": filtered_attrs}


def _filter_markdown_components_for_sharing(markdown: str) -> str:
    lines = markdown.replace("\r\n", "\n").replace("\r", "\n").split("\n")
    filtered_lines: list[str] = []
    line_index = 0

    while line_index < len(lines):
        if lines[line_index].strip().startswith("```"):
            code_block_end = _get_markdown_code_block_end(lines, line_index)
            filtered_lines.extend(lines[line_index:code_block_end])
            line_index = code_block_end
            continue

        component = _read_markdown_component_block(lines, line_index)
        if component is None:
            filtered_lines.append(lines[line_index])
            line_index += 1
            continue

        tag_name, _raw, next_line_index = component
        if tag_name in SHARED_NOTEBOOK_SUPPORTED_MARKDOWN_COMPONENT_TAGS:
            filtered_lines.append(_filter_supported_markdown_component_for_sharing(tag_name, _raw))
        else:
            filtered_lines.append(f"<{tag_name} />")
        line_index = next_line_index

    return "\n".join(filtered_lines)


def _filter_supported_markdown_component_for_sharing(tag_name: str, raw: str) -> str:
    supported_props = _SHARED_NOTEBOOK_MARKDOWN_COMPONENT_PROP_TYPES[tag_name]
    props = _parse_markdown_component_props(raw)
    filtered_props: dict[str, Any] = {}

    for prop_name, expected_type in supported_props.items():
        value = props.get(prop_name)
        if _is_markdown_component_prop_type(value, expected_type):
            if tag_name == "Embed" and prop_name == "src" and isinstance(value, str):
                value = value.strip()
                try:
                    _validate_notebook_embed_src(value)
                except ValidationError:
                    continue
            filtered_props[prop_name] = value

    return _serialize_markdown_component(tag_name, filtered_props)


def _is_markdown_component_prop_type(value: Any, expected_type: type | tuple[type, ...]) -> bool:
    if isinstance(value, bool):
        return expected_type is bool or (isinstance(expected_type, tuple) and bool in expected_type)
    if isinstance(value, expected_type):
        return True
    return False


def _serialize_markdown_component(tag_name: str, props: dict[str, Any]) -> str:
    prop_source = "".join(
        _serialize_markdown_component_prop(name, props[name])
        for name in _SHARED_NOTEBOOK_MARKDOWN_COMPONENT_PROP_TYPES[tag_name]
        if name in props
    )
    return f"<{tag_name}{prop_source} />"


def _serialize_markdown_component_prop(name: str, value: Any) -> str:
    if value is True:
        return f" {name}"
    if isinstance(value, str):
        return f" {name}={json.dumps(value, ensure_ascii=False)}"
    return f" {name}={{{json.dumps(value, ensure_ascii=False, separators=(',', ':'))}}}"


def _coerce_query_attr(raw: Any) -> dict[str, Any] | None:
    """Resolve a notebook ph-query node's `query` attribute to a dict.

    Notebooks save attrs through tiptap, which can serialize complex attrs as JSON strings
    (the `jsonAttr` wrapper in `NodeWrapper.tsx` round-trips through `JSON.stringify` /
    `JSON.parse`). Older notebooks were also saved with stringified queries before the
    `convertInsightQueryStringsToObjects` frontend migration normalized them. Accept either form.
    """
    if isinstance(raw, dict):
        return raw
    if isinstance(raw, str):
        try:
            parsed = json.loads(raw)
        except (TypeError, ValueError):
            return None
        return parsed if isinstance(parsed, dict) else None
    return None


def iter_prosemirror_nodes(doc: Any) -> Iterator[TipTapNode]:
    """Yield every node in a ProseMirror document, depth-first.

    Tolerates malformed input — anything that isn't a dict with a `content` list is skipped.
    """
    if not isinstance(doc, dict):
        return
    yield doc
    children = doc.get("content")
    if not isinstance(children, list):
        return
    for child in children:
        yield from iter_prosemirror_nodes(child)


def extract_referenced_insight_short_ids(content: Any) -> set[str]:
    """Walk a notebook's ProseMirror document and collect every saved-insight short_id it embeds.

    Only ``ph-query`` nodes whose query is a ``SavedInsightNode`` reference an insight by id.
    Inline (ad-hoc) queries store the full query in node attrs — see
    :func:`extract_inline_query_nodes` for those.
    """
    short_ids: set[str] = set()
    for node in iter_prosemirror_nodes(content):
        if node.get("type") != QUERY_NODE_TYPE:
            continue
        attrs = node.get("attrs")
        if not isinstance(attrs, dict):
            continue
        query = _coerce_query_attr(attrs.get("query"))
        if query is None:
            continue
        if query.get("kind") != SAVED_INSIGHT_NODE_KIND:
            continue
        short_id = query.get("shortId")
        if isinstance(short_id, str) and short_id:
            short_ids.add(short_id)
    for _node_id, query in iter_markdown_query_nodes(content):
        if query.get("kind") != SAVED_INSIGHT_NODE_KIND:
            continue
        short_id = query.get("shortId")
        if isinstance(short_id, str) and short_id:
            short_ids.add(short_id)
    return short_ids


def extract_inline_query_nodes(content: Any) -> list[tuple[str, dict[str, Any]]]:
    """Walk a notebook's ProseMirror document and collect every inline (non-saved-insight) query.

    Returns a list of ``(nodeId, query_dict)`` pairs. Used by the shared-notebook payload
    builder to pre-compute results for ad-hoc queries (DataTableNode, HogQLQuery, InsightVizNode
    without a saved insight reference, etc.) so the shared viewer can render them without
    POSTing to ``/api/projects/<id>/query/`` — a path sharing tokens cannot reach.

    Saved insights are deliberately excluded; they go through
    :func:`extract_referenced_insight_short_ids` and the existing
    ``InsightSerializer`` shared-mode path.

    Legacy ProseMirror ``ph-query`` nodes whose ``nodeId`` is missing are skipped; without it
    the frontend has no key to look the cached result up by. Markdown ``<Query>`` components
    without an explicit ``nodeId`` instead receive a content-derived stable ID (see
    :func:`iter_markdown_query_nodes`).
    """
    inline_nodes: list[tuple[str, dict[str, Any]]] = []
    for node in iter_prosemirror_nodes(content):
        if node.get("type") != QUERY_NODE_TYPE:
            continue
        attrs = node.get("attrs")
        if not isinstance(attrs, dict):
            continue
        query = _coerce_query_attr(attrs.get("query"))
        if query is None:
            continue
        if query.get("kind") == SAVED_INSIGHT_NODE_KIND:
            continue
        node_id = attrs.get("nodeId")
        if not isinstance(node_id, str) or not node_id:
            continue
        inline_nodes.append((node_id, query))
    for node_id, query in iter_markdown_query_nodes(content):
        if query.get("kind") == SAVED_INSIGHT_NODE_KIND:
            continue
        inline_nodes.append((node_id, query))
    return inline_nodes


def iter_markdown_query_nodes(content: Any) -> Iterator[tuple[str, dict[str, Any]]]:
    markdown = _get_markdown_notebook_markdown(content)
    if markdown is None:
        return

    occurrences: dict[str, int] = {}
    for tag_name, raw, _next_line_index in _iter_markdown_component_blocks(markdown):
        if tag_name != MARKDOWN_QUERY_TAG:
            continue

        props = _parse_markdown_component_props(raw)
        fingerprint = _get_markdown_component_fingerprint(tag_name, props)
        occurrence = occurrences.get(fingerprint, 0)
        occurrences[fingerprint] = occurrence + 1

        query = _coerce_query_attr(props.get("query"))
        if query is None:
            continue

        explicit_node_id = props.get("nodeId")
        node_id = (
            explicit_node_id
            if isinstance(explicit_node_id, str) and explicit_node_id
            else _create_stable_markdown_node_id(fingerprint, occurrence)
        )
        yield node_id, query


@frozen
class MarkdownBlock:
    """One addressable span of a markdown notebook, in document order.

    Blocks never overlap and never include the blank lines between them, so replacing the
    span `[start, end)` keeps the separators that group cells into cards.
    """

    kind: str
    tag_name: str | None
    node_id: str
    explicit_node_id: str | None
    source: str
    start: int
    end: int


def _get_anchored_markdown_cell_end(lines: list[str], start: int) -> int:
    end = start
    while end < len(lines):
        stripped = lines[end].strip()
        if _MARKDOWN_NODE_ANCHOR_REGEX.match(stripped) or _opens_markdown_component_block(lines, end):
            break
        if not stripped:
            if end + 1 >= len(lines) or not lines[end + 1].strip():
                break
            end += 1
            continue
        if stripped.startswith("```"):
            end = _get_markdown_code_block_end(lines, end)
        else:
            end += 1
    while end > start and not lines[end - 1].strip():
        end -= 1
    return end


def iter_markdown_blocks(
    markdown: str, max_prose_blocks: int | None = None, *, group_anchored_prose: bool = False
) -> Iterator[MarkdownBlock]:
    """Walk every block of a markdown notebook, prose included.

    A block whose document carries an anchor above it takes that stored id, which survives an
    edit to the block. A block without one falls back to an id derived from its own content, so
    the id changes when the block changes and a caller must resolve it against the read it edits.

    `group_anchored_prose` keeps an anchored markdown cell's paragraphs together until a card
    boundary, another anchor, or a component, so cell edits replace all of its prose.

    `max_prose_blocks` bounds how many prose blocks this builds. A save accepts a body up to
    `DATA_UPLOAD_MAX_MEMORY_SIZE`, and a body of that size holds millions of one-character
    blocks, so an unbounded walk lets one request allocate for minutes. The walk continues past
    the cap without building the extra prose, because component tags carry the cells that run
    and must stay addressable wherever they sit in the document.
    """
    lines = _split_markdown_lines(markdown)
    occurrences: dict[str, int] = {}
    line_index = 0
    prose_built = 0
    pending_anchor_id: str | None = None
    # An anchor names one block. A second copy of one reaches the document through a three-way
    # markdown merge, and two blocks under one id leave neither addressable, because every cell
    # tool refuses an id that names more than one. The first keeps it; the rest fall back to a
    # derived id. `ensureUniqueNodeIds` does the same in the editor.
    claimed_anchor_ids: set[str] = set()

    def claim_anchor_id() -> str | None:
        nonlocal pending_anchor_id
        anchor_id = pending_anchor_id
        pending_anchor_id = None
        if anchor_id is None or anchor_id in claimed_anchor_ids:
            return None
        claimed_anchor_ids.add(anchor_id)
        return anchor_id

    # Two counters over one walk: code points to slice the document Python holds, UTF-16 units
    # to report, because the caller slices in UTF-16. Each advances once per character.
    code_points = 0
    utf16 = 0

    def prose_budget_left() -> bool:
        return max_prose_blocks is None or prose_built < max_prose_blocks

    def span_code_points(start_line: int, end_line: int) -> int:
        """Width of lines `[start_line, end_line)`, each with the terminator that closes it."""
        width = 0
        for index in range(start_line, end_line):
            width += len(lines[index])
            width += _markdown_terminator_width(markdown, code_points + width)
        return width

    def block_source(start_line: int, end_line: int) -> str:
        # The last line of a document often has no terminator, so one looked up from the end of
        # the span would be a character of the block itself.
        width = span_code_points(start_line, end_line - 1) + len(lines[end_line - 1])
        return markdown[code_points : code_points + width]

    def consume(start_line: int, end_line: int) -> None:
        nonlocal code_points, utf16
        width = span_code_points(start_line, end_line)
        utf16 += _utf16_length(markdown[code_points : code_points + width])
        code_points += width

    while line_index < len(lines):
        if not lines[line_index].strip():
            # An anchor names only the block on the line below it. Text removed from under an
            # anchor leaves the anchor behind, and carried over a blank line it would hand the id
            # to the next block, so an edit by that id would reach a block its caller never read.
            pending_anchor_id = None
            consume(line_index, line_index + 1)
            line_index += 1
            continue

        # Read before the fence and tag scans, so an anchor never becomes a block of its own and
        # its span never joins the block it names. A fenced block is consumed whole below, so an
        # anchor written as an example inside one is never seen here.
        anchor_match = _MARKDOWN_NODE_ANCHOR_REGEX.match(lines[line_index].strip())
        if anchor_match:
            pending_anchor_id = anchor_match.group(1)
            consume(line_index, line_index + 1)
            line_index += 1
            continue

        if (
            group_anchored_prose
            and pending_anchor_id is not None
            and pending_anchor_id not in claimed_anchor_ids
            and not _opens_markdown_component_block(lines, line_index)
        ):
            end_line_index = _get_anchored_markdown_cell_end(lines, line_index)
            if prose_budget_left():
                prose_built += 1
                yield _build_markdown_prose_block(
                    block_source(line_index, end_line_index), utf16, occurrences, claim_anchor_id()
                )
            pending_anchor_id = None
            consume(line_index, end_line_index)
            line_index = end_line_index
            continue

        if lines[line_index].strip().startswith("```"):
            end_line_index = _get_markdown_code_block_end(lines, line_index)
            if prose_budget_left():
                prose_built += 1
                yield _build_markdown_prose_block(
                    block_source(line_index, end_line_index), utf16, occurrences, claim_anchor_id()
                )
            pending_anchor_id = None
            consume(line_index, end_line_index)
            line_index = end_line_index
            continue

        component = (
            _read_markdown_component_block(lines, line_index)
            if _opens_markdown_component_block(lines, line_index)
            else None
        )
        if component is not None:
            tag_name, raw, next_line_index = component
            yield _build_markdown_component_block(
                tag_name, raw, block_source(line_index, next_line_index), utf16, occurrences, claim_anchor_id()
            )
            consume(line_index, next_line_index)
            line_index = next_line_index
            continue

        end_line_index = line_index + 1
        while end_line_index < len(lines) and _continues_markdown_prose_block(lines, end_line_index):
            end_line_index += 1
        if prose_budget_left():
            prose_built += 1
            yield _build_markdown_prose_block(
                block_source(line_index, end_line_index), utf16, occurrences, claim_anchor_id()
            )
        pending_anchor_id = None
        consume(line_index, end_line_index)
        line_index = end_line_index


def _continues_markdown_prose_block(lines: list[str], line_index: int) -> bool:
    stripped = lines[line_index].strip()
    if not stripped or stripped.startswith("```"):
        return False
    # An anchor names the block below it. Absorbed into the paragraph above, it would sit inside
    # that block's reported source, and a caller slicing the span back would not match it.
    if _MARKDOWN_NODE_ANCHOR_REGEX.match(stripped):
        return False
    return not _opens_markdown_component_block(lines, line_index)


def _opens_markdown_component_block(lines: list[str], line_index: int) -> bool:
    """Whether a component block starts at this line.

    The `<` test runs first because it settles every prose line with one string comparison. The
    regex scan behind it costs enough to dominate the walk of a large document when every line
    pays it.
    """
    stripped = lines[line_index].lstrip()
    if not stripped.startswith("<") and not stripped.startswith("\\<"):
        return False
    return _read_markdown_component_block(lines, line_index) is not None


_MARKDOWN_NODE_ANCHOR_REGEX = re.compile(r"^<!--ph:(phb-[A-Za-z0-9._-]{1,124})-->$")
"""A block id the document stores, written on its own line above the block it names.

Mirrors `NODE_ANCHOR_REGEX` in frontend/src/lib/components/MarkdownNotebook/markdown.ts,
prefix included. A wider pattern here would read an authorial `<!--ph:note-->` as an anchor
that the editor reads as a comment, and the two layers would disagree on the block list.

A paragraph has no attribute to carry an id, so a comment is the only place one fits, and it
renders nowhere.
"""

_MARKDOWN_LINE_SPLIT_REGEX = re.compile(r"\r\n|\r|\n")


def _split_markdown_lines(markdown: str) -> list[str]:
    """Lines, split on the terminators `_iter_markdown_component_blocks` collapses before it splits.

    A walk that split on `\n` alone would read different block boundaries than the component
    walker, so a tag after a lone `\r` would be prose here and a live cell there.

    Only the lines are materialized, matching what the component walker already allocates. A
    terminator's width comes from `_markdown_terminator_width` at the offset the walk holds,
    because a second list of that length costs as much again on a document of millions of lines.
    """
    return _MARKDOWN_LINE_SPLIT_REGEX.split(markdown)


def _markdown_terminator_width(markdown: str, offset: int) -> int:
    if offset >= len(markdown):
        return 0
    return 2 if markdown.startswith("\r\n", offset) else 1


def _utf16_length(text: str) -> int:
    """Length in UTF-16 code units, the unit the collaboration protocol and JavaScript both use.

    Python counts code points, so an astral character is one here and two there. Reporting code
    points would put every offset after an emoji two apart from where a JavaScript caller slices.
    """
    return len(text) + sum(1 for character in text if ord(character) > 0xFFFF)


def _build_markdown_prose_block(
    source: str, start: int, occurrences: dict[str, int], anchor_id: str | None = None
) -> MarkdownBlock:
    occurrence = occurrences.get(source, 0)
    occurrences[source] = occurrence + 1
    return MarkdownBlock(
        kind="prose",
        tag_name=None,
        node_id=anchor_id or _create_stable_markdown_prose_id(source, occurrence),
        explicit_node_id=anchor_id,
        source=source,
        start=start,
        end=start + _utf16_length(source),
    )


def _build_markdown_component_block(
    tag_name: str,
    raw: str,
    source: str,
    start: int,
    occurrences: dict[str, int],
    anchor_id: str | None = None,
) -> MarkdownBlock:
    props = _parse_markdown_component_props(raw)
    fingerprint = _get_markdown_component_fingerprint(tag_name, props)
    occurrence = occurrences.get(fingerprint, 0)
    occurrences[fingerprint] = occurrence + 1
    explicit_node_id = props.get("nodeId")
    explicit_node_id = explicit_node_id if isinstance(explicit_node_id, str) and explicit_node_id else None
    # The tag's own prop wins over an anchor above it: a cell added over MCP carries its id
    # there, and the editor writes no anchor for a tag that already has one.
    explicit_node_id = explicit_node_id or anchor_id
    return MarkdownBlock(
        kind="component",
        tag_name=tag_name,
        node_id=explicit_node_id or _create_stable_markdown_node_id(fingerprint, occurrence),
        explicit_node_id=explicit_node_id,
        source=source,
        start=start,
        end=start + _utf16_length(source),
    )


def _get_markdown_notebook_markdown(content: Any) -> str | None:
    if not isinstance(content, dict):
        return None

    nodes = content.get("content")
    if not isinstance(nodes, list) or len(nodes) != 1:
        return None

    node = nodes[0]
    if not isinstance(node, dict) or node.get("type") != MARKDOWN_NOTEBOOK_NODE_TYPE:
        return None

    attrs = node.get("attrs")
    markdown = attrs.get("markdown") if isinstance(attrs, dict) else None
    return markdown if isinstance(markdown, str) else None


def _iter_markdown_component_blocks(markdown: str) -> Iterator[tuple[str, str, int]]:
    lines = markdown.replace("\r\n", "\n").replace("\r", "\n").split("\n")
    line_index = 0

    while line_index < len(lines):
        if lines[line_index].strip().startswith("```"):
            line_index = _get_markdown_code_block_end(lines, line_index)
            continue

        component = _read_markdown_component_block(lines, line_index)
        if component is None:
            line_index += 1
            continue

        yield component
        line_index = component[2]


def _get_markdown_code_block_end(lines: list[str], line_index: int) -> int:
    next_line_index = line_index + 1
    while next_line_index < len(lines):
        if lines[next_line_index].strip().startswith("```"):
            return next_line_index + 1
        next_line_index += 1
    return next_line_index


def _read_markdown_component_block(lines: list[str], line_index: int) -> tuple[str, str, int] | None:
    first_line = lines[line_index].strip()
    recover_escaped_source = bool(_MARKDOWN_ESCAPED_COMPONENT_START_REGEX.match(first_line))
    if recover_escaped_source:
        first_line = _unescape_markdown_inline_source(first_line)
    if not _MARKDOWN_COMPONENT_START_REGEX.match(first_line):
        return None

    tag_match = re.match(r"^<([A-Z][A-Za-z0-9]*)", first_line)
    tag_name = tag_match.group(1) if tag_match else None
    if not tag_name:
        return None

    scan = _scan_markdown_component_block(lines, line_index, tag_name, recover_escaped_source)
    if recover_escaped_source and scan.next_line_index <= line_index + 1:
        return None
    if not scan.found_terminator:
        if _MARKDOWN_COMPONENT_TAG_REGEX.match(first_line):
            return tag_name, first_line, line_index + 1
        return None

    return tag_name, scan.raw, scan.next_line_index


def _scan_markdown_component_block(
    lines: list[str], line_index: int, tag_name: str, recover_escaped_source: bool = False
) -> _MarkdownComponentScan:
    raw_lines: list[str] = []
    state = _MarkdownComponentScanState()
    character_count = 0
    next_line_index = line_index
    fallback_raw_line_count: int | None = None
    fallback_next_line_index: int | None = None
    line_limit = min(len(lines), line_index + _MAX_MARKDOWN_COMPONENT_LINES)

    while next_line_index < line_limit:
        line = (
            _unescape_markdown_inline_source(lines[next_line_index])
            if recover_escaped_source
            else lines[next_line_index]
        )
        if next_line_index > line_index and not line.strip() and fallback_next_line_index is None:
            fallback_raw_line_count = len(raw_lines)
            fallback_next_line_index = next_line_index
        if next_line_index > line_index and not line.strip() and state.quote is None and state.expression_depth == 0:
            break

        separator_length = 1 if raw_lines else 0
        if raw_lines and character_count + separator_length + len(line) > _MAX_MARKDOWN_COMPONENT_CHARACTERS:
            break
        character_count += separator_length + len(line)
        raw_lines.append(line)

        character_index = 0
        while character_index < len(line):
            character = line[character_index]

            if state.opening_tag_closed:
                closing_tag = f"</{tag_name}>"
                if (
                    line.startswith(closing_tag, character_index)
                    and not line[character_index + len(closing_tag) :].strip()
                ):
                    return _MarkdownComponentScan(
                        raw="\n".join(raw_lines).strip(),
                        next_line_index=next_line_index + 1,
                        found_terminator=True,
                    )
                character_index += 1
                continue

            if (
                state.quote is None
                and state.expression_depth == 0
                and line.startswith("/>", character_index)
                and not line[character_index + 2 :].strip()
            ):
                return _MarkdownComponentScan(
                    raw="\n".join(raw_lines).strip(),
                    next_line_index=next_line_index + 1,
                    found_terminator=True,
                )
            _advance_markdown_component_scan(state, character)
            character_index += 1

        # Joined source contains a newline here. It consumes a pending escape without closing
        # the quoted value, matching the prop parser's treatment of backslash-newline.
        if state.quote is not None and state.escape_next:
            state.escape_next = False
        next_line_index += 1

    fallback_raw_lines = raw_lines if fallback_raw_line_count is None else raw_lines[:fallback_raw_line_count]
    return _MarkdownComponentScan(
        raw="\n".join(fallback_raw_lines).strip(),
        next_line_index=fallback_next_line_index if fallback_next_line_index is not None else next_line_index,
        found_terminator=False,
    )


def _unescape_markdown_inline_source(source: str) -> str:
    unescaped: list[str] = []
    index = 0
    while index < len(source):
        character = source[index]
        next_character = source[index + 1] if index + 1 < len(source) else None
        if character == "\\" and next_character in _MARKDOWN_INLINE_ESCAPABLE_CHARACTERS:
            unescaped.append(next_character)
            index += 2
        else:
            unescaped.append(character)
            index += 1
    return "".join(unescaped)


def _advance_markdown_component_scan(state: _MarkdownComponentScanState, character: str) -> None:
    if state.quote is not None:
        _advance_markdown_component_quote(state, character)
        return

    if state.expression_depth > 0:
        _advance_markdown_component_expression(state, character)
        return

    if state.awaiting_prop_value:
        if character.isspace():
            return
        state.awaiting_prop_value = False
        if character in {"'", '"'}:
            state.quote = character
            return
        if character == "{":
            state.expression_depth = 1
            return

    if character == "=":
        state.awaiting_prop_value = True
    elif character == ">":
        state.opening_tag_closed = True


def _advance_markdown_component_quote(state: _MarkdownComponentScanState, character: str) -> None:
    if state.escape_next:
        state.escape_next = False
    elif character == "\\":
        state.escape_next = True
    elif character == state.quote:
        state.quote = None


def _advance_markdown_component_expression(state: _MarkdownComponentScanState, character: str) -> None:
    if character in {"'", '"'}:
        state.quote = character
    elif character == "{":
        state.expression_depth += 1
    elif character == "}":
        state.expression_depth -= 1


def _parse_markdown_component_props(raw: str) -> dict[str, Any]:
    match = _MARKDOWN_COMPONENT_TAG_REGEX.match(raw)
    if not match:
        return {}

    props: dict[str, Any] = {}
    source = match.group(2) or ""
    index = 0
    while index < len(source):
        index = _skip_markdown_component_whitespace(source, index)
        if index >= len(source):
            break

        name_match = _MARKDOWN_COMPONENT_PROP_NAME_REGEX.match(source[index:])
        if not name_match:
            break

        name = name_match.group(1)
        index += len(name)
        index = _skip_markdown_component_whitespace(source, index)

        if index >= len(source) or source[index] != "=":
            props[name] = True
            continue

        index += 1
        index = _skip_markdown_component_whitespace(source, index)

        value, index = _read_markdown_component_prop_value(source, index)
        if _is_markdown_notebook_prop_value(value):
            props[name] = value

    return props


def _skip_markdown_component_whitespace(source: str, index: int) -> int:
    while index < len(source) and source[index].isspace():
        index += 1
    return index


def _read_markdown_component_prop_value(source: str, index: int) -> tuple[Any, int]:
    first_char = source[index] if index < len(source) else ""

    if first_char in {"'", '"'}:
        quote = first_char
        next_index = index + 1
        value = ""
        while next_index < len(source):
            character = source[next_index]
            if character == "\\" and next_index + 1 < len(source):
                value += source[next_index + 1]
                next_index += 2
                continue
            if character == quote:
                if quote == '"':
                    try:
                        parsed_value = json.loads(source[index : next_index + 1])
                        if isinstance(parsed_value, str):
                            return html.unescape(parsed_value), next_index + 1
                    except (TypeError, ValueError):
                        pass
                return html.unescape(value), next_index + 1
            value += character
            next_index += 1
        return None, next_index

    if first_char == "{":
        balanced = _read_balanced_markdown_expression(source, index)
        if balanced is None:
            return None, len(source)
        value, next_index = balanced
        return _parse_markdown_expression_value(value), next_index

    raw_match = _MARKDOWN_COMPONENT_RAW_PROP_VALUE_REGEX.match(source[index:])
    raw = raw_match.group(1) if raw_match else ""
    return _parse_markdown_expression_value(raw), index + len(raw)


def _read_balanced_markdown_expression(source: str, index: int) -> tuple[str, int] | None:
    depth = 0
    next_index = index
    quote: str | None = None

    while next_index < len(source):
        character = source[next_index]
        if quote:
            if character == quote and not _is_escaped_markdown_expression_quote(source, next_index):
                quote = None
            next_index += 1
            continue

        if character in {"'", '"'}:
            quote = character
            next_index += 1
            continue

        if character == "{":
            depth += 1
        if character == "}":
            depth -= 1
            if depth == 0:
                return source[index : next_index + 1], next_index + 1
        next_index += 1

    return None


def _is_escaped_markdown_expression_quote(source: str, quote_index: int) -> bool:
    backslash_count = 0
    index = quote_index - 1
    while index >= 0 and source[index] == "\\":
        backslash_count += 1
        index -= 1
    return backslash_count % 2 == 1


def _parse_markdown_expression_value(raw: str) -> Any:
    trimmed = raw.strip()
    unwrapped = trimmed[1:-1].strip() if trimmed.startswith("{") and trimmed.endswith("}") else trimmed

    if unwrapped == "true":
        return True
    if unwrapped == "false":
        return False
    if unwrapped == "null":
        return None
    if _MARKDOWN_COMPONENT_NUMBER_REGEX.match(unwrapped):
        return float(unwrapped) if "." in unwrapped else int(unwrapped)

    try:
        return json.loads(unwrapped)
    except (TypeError, ValueError):
        return trimmed


def _is_markdown_notebook_prop_value(value: Any) -> bool:
    if value is None or isinstance(value, str | int | float | bool):
        return True
    if isinstance(value, list):
        return all(_is_markdown_notebook_prop_value(item) for item in value)
    if isinstance(value, dict):
        return all(isinstance(key, str) and _is_markdown_notebook_prop_value(item) for key, item in value.items())
    return False


def _get_markdown_component_fingerprint(tag_name: str, props: dict[str, Any]) -> str:
    return json.dumps(
        {"type": "component", "tagName": tag_name, "props": _sort_markdown_component_props(props)},
        ensure_ascii=False,
        separators=(",", ":"),
    )


def _sort_markdown_component_props(props: dict[str, Any]) -> dict[str, Any]:
    return {key: _sort_markdown_component_prop_value(props[key]) for key in sorted(props)}


def _sort_markdown_component_prop_value(value: Any) -> Any:
    if isinstance(value, list):
        return [_sort_markdown_component_prop_value(item) for item in value]
    if isinstance(value, dict):
        return _sort_markdown_component_props(value)
    return value


def _create_stable_markdown_node_id(fingerprint: str, occurrence: int) -> str:
    return f"mdn-{_hash_markdown_node_id_seed(fingerprint)}-{occurrence}"


def _create_stable_markdown_prose_id(source: str, occurrence: int) -> str:
    # A separate prefix from `mdn-` keeps prose ids and component ids in disjoint spaces, so a
    # caller can route an id to the right lookup without inspecting the document.
    #
    # This does not reuse `_hash_markdown_node_id_seed`, whose 32 bits mirror the frontend's own
    # hash for component ids. Two prose blocks that collide there share an id, and an edit meant
    # for one lands on the other, so prose takes a width where a collision cannot be constructed.
    digest = hashlib.blake2b(source.encode("utf-8"), digest_size=8).digest()
    return f"mdp-{_to_base36(int.from_bytes(digest, 'big'))}-{occurrence}"


def _hash_markdown_node_id_seed(value: str) -> str:
    hash_value = 5381
    encoded = value.encode("utf-16-le", "surrogatepass")
    for index in range(0, len(encoded), 2):
        code_unit = encoded[index] | (encoded[index + 1] << 8)
        hash_value = ((hash_value * 33) ^ code_unit) & 0xFFFFFFFF
    return _to_base36(hash_value)


def _to_base36(value: int) -> str:
    digits = "0123456789abcdefghijklmnopqrstuvwxyz"
    if value == 0:
        return "0"
    result = ""
    while value:
        value, remainder = divmod(value, 36)
        result = digits[remainder] + result
    return result


def create_bullet_list(items: list[str] | list[TipTapContent] | TipTapContent) -> TipTapNode:
    """Create a bullet list with list items. Items can be strings or content arrays."""
    list_items = []
    for item in items:
        if isinstance(item, str):
            list_items.append({"type": "listItem", "content": [create_paragraph_with_text(item)]})
        elif isinstance(item, list):
            # item is already a content array (could be paragraph + nested list)
            list_items.append({"type": "listItem", "content": item})
        else:
            # item is a single content node
            list_items.append({"type": "listItem", "content": [create_paragraph_with_content([item])]})

    return {"type": "bulletList", "content": list_items}


def create_heading_with_text(text: str, level: int, *, collapsed: bool = False) -> TipTapNode:
    """Create a heading node with sanitized text content."""
    heading_id = str(uuid.uuid4())
    return {
        "type": "heading",
        "attrs": {"id": heading_id, "level": level, "data-toc-id": heading_id, "collapsed": collapsed},
        "content": [{"type": "text", "text": sanitize_text_content(text)}],
    }


def create_paragraph_with_text(text: str, marks: list[dict[str, Any]] | None = None) -> TipTapNode:
    """Create a paragraph node with sanitized text content and optional marks."""
    content_node: dict[str, Any] = {"type": "text", "text": sanitize_text_content(text)}
    if marks:
        content_node["marks"] = marks
    return {
        "type": "paragraph",
        "content": [content_node],
    }


def create_paragraph_with_content(content: TipTapContent) -> TipTapNode:
    """Create a paragraph node with a list of content items."""
    return {
        "type": "paragraph",
        "content": content,
    }


def create_text_content(text: str, is_bold: bool = False, is_italic: bool = False) -> TipTapNode:
    """Create a text node with optional marks."""
    node: dict[str, Any] = {"type": "text", "text": text}
    marks = []
    if is_bold:
        marks.append({"type": "bold"})
    if is_italic:
        marks.append({"type": "italic"})
    if marks:
        node["marks"] = marks
    return node


def create_empty_paragraph() -> TipTapNode:
    """Create a paragraph node with no content to add spacing."""
    return {"type": "paragraph"}


def create_task_list(items: list[tuple[str, bool]]) -> TipTapNode:
    """Create a bullet list with checkbox-style items.

    Args:
        items: List of tuples (task_text, is_completed)
    """
    list_items = []
    for task_text, is_completed in items:
        checkbox = "[x]" if is_completed else "[ ]"
        task_content = f"{checkbox} {task_text}"
        list_items.append({"type": "listItem", "content": [create_paragraph_with_text(task_content)]})

    return {"type": "bulletList", "content": list_items}


def sanitize_text_content(text: str) -> str:
    """Sanitize text content to ensure it's valid for TipTap editor."""
    if not text or not text.strip():
        raise ValueError("Empty text should not be passed to create heading or paragraph")
    return text.strip()