Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

6.2 KB
Raw Blame
# Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
import logging
import time
from collections.abc import Mapping
from types import FrameType
from typing import Any

from django.db import transaction
from typing_extensions import override

from zerver.actions.message_edit import do_update_embedded_data
from zerver.actions.message_send import (
    do_send_url_embed_data_event,
    is_latest_preview_draft,
    render_incoming_message,
    render_unsaved_message,
)
from zerver.lib.cache import (
    cache_delete,
    pending_preview_draft_cache_key,
    preview_draft_content_hash,
)
from zerver.lib.mention import MentionBackend, MentionData
from zerver.lib.url_preview import preview as url_preview
from zerver.lib.url_preview.types import UrlEmbedData
from zerver.models import Message, Realm, UserProfile
from zerver.models.users import get_user_profile_by_id
from zerver.worker.base import InterruptConsumeError, QueueProcessingWorker, assign_queue

logger = logging.getLogger(__name__)


def fetch_link_embed_data(url: str) -> UrlEmbedData | None:
    start_time = time.time()
    link_embed_data = url_preview.get_link_embed_data(url)
    logging.info("Time spent on get_link_embed_data for %s: %s", url, time.time() - start_time)
    return link_embed_data


@assign_queue("embed_links")
class FetchLinksEmbedData(QueueProcessingWorker):
    # This is a slow queue with network requests, so a disk write is negligible.
    # Update stats file after every consume call.
    CONSUME_ITERATIONS_BEFORE_UPDATE_STATS_NUM = 1

    @override
    def consume(self, event: Mapping[str, Any]) -> None:
        if event.get("type") == "url_embed_data":
            self.handle_url_embed_data_request(
                user_profile_id=event["user_id"],
                content=event["content"],
                cached_urls=event["cached_urls"],
                urls=event["urls"],
            )
            return

        url_embed_data = {url: fetch_link_embed_data(url) for url in event["urls"]}

        # Ideally, we should use `durable=True` here. However, in the
        # `test_message_update_race_condition` test, this function is not called
        # as the outermost transaction. As a result, it's acceptable to make an
        # exception and not use `durable=True` in this case.
        #
        # For more details on why the `consume` method is called directly in tests, see:
        # https://zulip.readthedocs.io/en/latest/subsystems/queuing.html#publishing-events-into-a-queue
        with transaction.atomic(savepoint=False):
            try:
                message = Message.objects.select_for_update(no_key=True).get(id=event["message_id"])
            except Message.DoesNotExist:
                # Message may have been deleted
                return

            # If the message changed, we will run this task after updating the message
            # in zerver.actions.message_edit.check_update_message
            if message.content != event["message_content"]:
                return

            # Fetch the realm whose settings we're using for rendering
            realm = Realm.objects.get(id=event["message_realm_id"])

            # If rendering fails, the called code will raise a JsonableError.
            mention_data = MentionData(
                mention_backend=MentionBackend(message.realm_id),
                content=message.content,
                message_sender=message.sender,
            )
            rendering_result = render_incoming_message(
                message,
                message.content,
                realm,
                url_embed_data=url_embed_data,
                mention_data=mention_data,
            )
            do_update_embedded_data(message.sender, message, rendering_result, mention_data)

    def handle_url_embed_data_request(
        self, *, user_profile_id: int, content: str, cached_urls: list[str], urls: list[str]
    ) -> None:
        try:
            sender = get_user_profile_by_id(user_profile_id)
        except UserProfile.DoesNotExist:
            # The user may have been deleted.
            return

        content_hash = preview_draft_content_hash(content)
        try:
            fetched_url_embed_data: dict[str, UrlEmbedData | None] = {}
            unavailable_urls = url_preview.get_unavailable_preview_urls(urls)
            for url in urls:
                if url in unavailable_urls:
                    continue
                if not is_latest_preview_draft(sender.id, content_hash):
                    return
                try:
                    fetched_url_embed_data[url] = fetch_link_embed_data(url)
                finally:
                    if fetched_url_embed_data.get(url) is None:
                        url_preview.mark_preview_url_unavailable(url)

            if not is_latest_preview_draft(sender.id, content_hash):
                return
            previewed_url_embed_data = {url: fetch_link_embed_data(url) for url in cached_urls}
            rendered_content = render_unsaved_message(
                sender,
                content,
                url_embed_data={**previewed_url_embed_data, **fetched_url_embed_data},
            ).rendered_content
            previewed_rendered_content = render_unsaved_message(
                sender, content, url_embed_data=previewed_url_embed_data
            ).rendered_content
            if rendered_content != previewed_rendered_content:
                do_send_url_embed_data_event(sender, content, rendered_content)
        finally:
            # Lets a later render queue a job for any links this one didn't fetch.
            cache_delete(pending_preview_draft_cache_key(sender.id, content_hash))

    @override
    def timer_expired(
        self, limit: int, events: list[dict[str, Any]], signal: int, frame: FrameType | None
    ) -> None:
        assert len(events) == 1
        event = events[0]

        if event.get("type") == "url_embed_data":
            fetching_for = f"a preview by user {event['user_id']}"
        else:
            fetching_for = f"message {event['message_id']}"
        logging.warning(
            "Timed out in %s after %s seconds while fetching URLs for %s: %s",
            self.queue_name,
            limit,
            fetching_for,
            event["urls"],
        )
        raise InterruptConsumeError