Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.1 KB
Raw Blame
from collections.abc import Callable
from datetime import timedelta
from typing import Any, cast
from uuid import UUID

from django.db import transaction
from django.db.models import Q, QuerySet

from posthog.models import Team, User
from posthog.models.activity_logging.activity_log import changes_between
from posthog.models.comment import Comment
from posthog.models.utils import UUIDT

from products.notebooks.backend import markdown_collab
from products.notebooks.backend.activity_logging import log_notebook_activity
from products.notebooks.backend.facade.contracts import (
    MarkdownNotebookMigrationPreview,
    MarkdownNotebookMigrationResult,
    MarkdownNotebookMigrationStats,
    NotebookContentNotConvertible,
)
from products.notebooks.backend.markdown_conversion import (
    JSONContent,
    NotebookMarkdownConversionOptions,
    build_markdown_notebook_content,
    convert_notebook_content_to_markdown,
    get_markdown_notebook_markdown,
    is_markdown_notebook_content,
    notebook_content_has_comment_marks,
)
from products.notebooks.backend.models import Notebook

MAX_NOTEBOOK_MIGRATION_BATCH_SIZE = 500


def get_markdown_notebook_migration_stats(team_id: int | None = None) -> MarkdownNotebookMigrationStats:
    _validate_team_id(team_id)
    queryset = _notebook_scope(team_id)
    total = queryset.count()
    converted = queryset.filter(content__content__0__type=markdown_collab.MARKDOWN_NOTEBOOK_NODE_TYPE).count()
    return MarkdownNotebookMigrationStats(total=total, converted=converted, pending=total - converted, team_id=team_id)


def migrate_notebooks_to_markdown(
    *,
    user: User,
    team_id: int | None = None,
    dry_run: bool = True,
    batch_size: int | None = None,
    max_previews: int = 5,
) -> MarkdownNotebookMigrationResult:
    batch_size = _validate_batch_size(batch_size)
    stats = get_markdown_notebook_migration_stats(team_id)
    converted = 0
    skipped = 0
    errored = 0
    previews: list[MarkdownNotebookMigrationPreview] = []
    errors: list[str] = []

    queryset = _pending_notebook_scope(team_id).select_related("team").order_by("team_id", "id")
    if batch_size is not None:
        queryset = queryset[:batch_size]

    for notebook in queryset.iterator(chunk_size=100):
        try:
            next_content = to_markdown_notebook_content(
                notebook.content,
                organization_id=notebook.team.organization_id,
                comment_replies_by_mark_id=_build_comment_replies_by_mark_id(notebook),
            )
            if next_content is None:
                skipped += 1
                continue
            markdown = get_markdown_notebook_markdown(next_content)
            if dry_run:
                converted += 1
                if len(previews) < max_previews:
                    previews.append(
                        MarkdownNotebookMigrationPreview(
                            short_id=str(notebook.short_id),
                            title=notebook.title,
                            before_version=notebook.version,
                            markdown_preview=markdown[:500],
                        )
                    )
                continue

            if _convert_notebook(notebook, user=user, content=next_content, text_content=markdown):
                converted += 1
            else:
                skipped += 1
        except Exception as err:
            errored += 1
            errors.append(f"{notebook.short_id}: {err}")

    final_stats = stats if dry_run else get_markdown_notebook_migration_stats(team_id)
    return MarkdownNotebookMigrationResult(
        dry_run=dry_run,
        team_id=team_id,
        batch_size=batch_size,
        total=stats.total,
        already_converted=stats.converted,
        pending_before=stats.pending,
        pending_after=final_stats.pending,
        converted=converted,
        skipped=skipped,
        errored=errored,
        previews=previews,
        errors=errors[:20],
    )


def _validate_team_id(team_id: int | None) -> None:
    if team_id is not None and not Team.objects.filter(id=team_id).exists():
        raise ValueError(f"Team {team_id} does not exist")


def _validate_batch_size(batch_size: int | None) -> int | None:
    if batch_size is None:
        return None
    if batch_size < 1:
        raise ValueError("Batch size must be at least 1")
    if batch_size > MAX_NOTEBOOK_MIGRATION_BATCH_SIZE:
        raise ValueError(f"Batch size must be {MAX_NOTEBOOK_MIGRATION_BATCH_SIZE} or less")
    return batch_size


def _notebook_scope(team_id: int | None) -> QuerySet[Notebook]:
    queryset = Notebook.objects.all()
    return queryset.filter(team_id=team_id) if team_id is not None else queryset


def _pending_notebook_scope(team_id: int | None) -> QuerySet[Notebook]:
    return _notebook_scope(team_id).filter(
        Q(content__content__0__type__isnull=True)
        | ~Q(content__content__0__type=markdown_collab.MARKDOWN_NOTEBOOK_NODE_TYPE)
    )


def _convert_notebook(notebook: Notebook, *, user: User, content: dict[str, Any], text_content: str) -> bool:
    with transaction.atomic():
        locked_notebook = Notebook.objects.select_for_update().select_related("team").get(pk=notebook.pk)
        if is_markdown_notebook_content(locked_notebook.content):
            return False

        before_update = Notebook.objects.select_related("created_by", "last_modified_by").get(pk=locked_notebook.pk)
        locked_notebook.content = content
        locked_notebook.text_content = text_content
        locked_notebook.version += 1
        locked_notebook.save(
            update_fields=[
                "content",
                "text_content",
                "version",
            ]
        )

        notify_team_id = locked_notebook.team_id
        notify_notebook_id = str(locked_notebook.short_id)
        notify_version = locked_notebook.version
        transaction.on_commit(
            lambda: markdown_collab.publish_notebook_update(
                notify_team_id,
                notify_notebook_id,
                notify_version,
                diff=None,
            )
        )

    changes = changes_between("Notebook", previous=before_update, current=locked_notebook)
    log_notebook_activity(
        activity="updated",
        notebook=locked_notebook,
        organization_id=cast(UUIDT, locked_notebook.team.organization_id),
        team_id=locked_notebook.team_id,
        user=before_update.last_modified_by or before_update.created_by or user,
        was_impersonated=False,
        changes=changes,
        created_at=before_update.last_modified_at + timedelta(seconds=1),
    )
    return True


def _build_comment_replies_by_mark_id(notebook: Notebook) -> dict[str, list[Any]]:
    if not notebook_content_has_comment_marks(notebook.content):
        return {}

    root_comments = list(
        Comment.objects.filter(team_id=notebook.team_id, scope="Notebook", item_id=notebook.short_id)
        .filter(deleted=False, source_comment_id__isnull=True)
        .select_related("created_by")
        .order_by("created_at")
    )
    root_ids = [comment.id for comment in root_comments]
    replies_by_source_id: dict[Any, list[Comment]] = {comment.id: [] for comment in root_comments}
    if root_ids:
        for reply in (
            Comment.objects.filter(team_id=notebook.team_id, source_comment_id__in=root_ids)
            .filter(deleted=False)
            .select_related("created_by")
            .order_by("created_at")
        ):
            if isinstance(reply.item_context, dict) and reply.item_context.get("is_emoji"):
                continue
            replies_by_source_id.setdefault(reply.source_comment_id, []).append(reply)

    replies_by_mark_id: dict[str, list[Any]] = {}
    for comment in root_comments:
        item_context = comment.item_context
        mark_id = (
            item_context.get("id") if isinstance(item_context, dict) and item_context.get("type") == "mark" else None
        )
        if not isinstance(mark_id, str):
            continue
        thread = sorted([comment, *replies_by_source_id.get(comment.id, [])], key=lambda item: item.created_at)
        replies_by_mark_id[mark_id] = [_comment_to_reply(comment) for comment in thread]
    return replies_by_mark_id


def _comment_to_reply(comment: Comment) -> dict[str, Any]:
    author = None
    if comment.created_by:
        author = comment.created_by.first_name or comment.created_by.email
    return {
        "id": str(comment.id),
        "text": comment.content or "",
        **({"author": author} if author else {}),
        **({"authorId": comment.created_by_id} if comment.created_by_id else {}),
        "at": comment.created_at.isoformat(),
    }


def to_markdown_notebook_content(
    content: Any,
    *,
    organization_id: UUID,
    comment_replies_by_mark_id: dict[str, list[Any]] | None = None,
) -> JSONContent | None:
    """Convert a rich-text notebook document into a markdown notebook document.

    Returns None when the content already is a markdown notebook document. The admin migration and
    notebook creation both call this, so a notebook created from rich text matches a migrated one.
    Raises NotebookContentNotConvertible when the document is malformed, for example when a node's
    marks are not a list.
    """
    if is_markdown_notebook_content(content):
        return None
    try:
        # The mention scan walks the same caller-supplied tree as the converter, so its failures map to the same error.
        options = NotebookMarkdownConversionOptions(
            comment_replies_by_mark_id=comment_replies_by_mark_id,
            get_mention_label=_build_mention_label_getter(content, organization_id),
        )
        markdown = convert_notebook_content_to_markdown(content, options)
    except (AttributeError, KeyError, RecursionError, TypeError, ValueError) as err:
        raise NotebookContentNotConvertible(
            "Notebook content is not a valid rich-text document, so it cannot be stored as a markdown notebook."
        ) from err
    return build_markdown_notebook_content(markdown)


def _build_mention_label_getter(content: Any, organization_id: UUID) -> Callable[[int], str | None]:
    user_ids = _collect_mention_user_ids(content)
    if not user_ids:
        return lambda user_id: None

    users = User.objects.filter(
        id__in=user_ids,
        organization_membership__organization_id=organization_id,
    ).only("id", "first_name", "email")
    labels_by_user_id = {
        user.id: f"@{user.first_name or user.email}" for user in users if user.first_name or user.email
    }
    return lambda user_id: labels_by_user_id.get(user_id)


def _collect_mention_user_ids(content: Any) -> set[int]:
    user_ids: set[int] = set()

    def visit(node: Any) -> None:
        if isinstance(node, dict):
            if node.get("type") == "ph-mention":
                attrs = node.get("attrs")
                user_id = attrs.get("id") if isinstance(attrs, dict) else None
                if isinstance(user_id, int):
                    user_ids.add(user_id)
            for value in node.values():
                visit(value)
        elif isinstance(node, list):
            for child in node:
                visit(child)

    visit(content)
    return user_ids