Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

19.8 KB
Raw Blame
import contextlib
import re
from datetime import datetime, timezone
from itertools import zip_longest
from typing import Any, Literal, TypeAlias, TypedDict, cast

import regex
from django.core.exceptions import ValidationError
from requests.utils import requote_uri

from zerver.lib.timestamp import datetime_to_global_time
from zerver.lib.types import Validator
from zerver.lib.validator import (
    WildValue,
    check_dict,
    check_int,
    check_list,
    check_string,
    check_string_in,
    check_url,
)

# stubs
ZerverFieldsT: TypeAlias = dict[str, Any]
SlackToZulipUserIDT: TypeAlias = dict[str, int]
AddedChannelsT: TypeAlias = dict[str, tuple[str, int]]
SlackFieldsT: TypeAlias = dict[str, Any]

# Slack link can be in the format <http://www.foo.com|www.foo.com> and <http://foo.com/>
LINK_REGEX = r"""
              (<)                                                              # match '>'
              (http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/|ftp:\/\/)?  # protocol and www
                  ([a-z0-9]+([\-\.]{1}[a-z0-9]+)*)(\.)                         # domain name
                      ([a-z]{2,63}(:[0-9]{1,5})?)                              # domain
                  (\/[^>]*)?                                                   # path
              (\|)?(?:\|([^>]+))?                                # char after pipe (for Slack links)
              (>)
              """

SLACK_MAILTO_REGEX = r"""
                      <((mailto:)?                     # match  `<mailto:`
                      ([\w\.-]+@[\w\.-]+(\.[\w]+)+))   # match email
                          (\|)?                        # match pipe
                      ([\w\.-]+@[\w\.-]+(\.[\w]+)+)?>  # match email
                      """

SLACK_USERMENTION_REGEX = r"""
                           (<@)                  # Start with '<@'
                               ([a-zA-Z0-9]+)    # Here we have the Slack id
                           (\|)?                 # We not always have a vertical line in mention
                               ([a-zA-Z0-9]+)?   # If vertical line is present, this is short name
                           (>)                   # ends with '>'
                           """
# Slack doesn't have mid-word message-formatting like Zulip.
# Hence, ~stri~ke doesn't format the word in Slack, but ~~stri~~ke
# formats the word in Zulip
SLACK_STRIKETHROUGH_REGEX = r"""
                             (
                                # Capture punctuation (\p{P}), white space (\p{Zs}),
                                # symbols (\p{S}) or newline.
                                # Skip ~ to not reformat the same string twice
                                # Skip @ and \
                                # Skip closing brackets & closing quote (\p{Pf}\p{Pe})
                                (?![~`@\\\p{Pf}\p{Pe}])
                                [\p{P}\p{Zs}\p{S}]|^
                             )
                             (\~)                                  # followed by a ~
                                 ([^~]+)                           # any character except ~
                             (\~)                                  # followed by a ~
                             (
                                # Capture punctuation, white space, symbols or end of
                                # line.
                                # Skip ~ to not reformat the same string twice
                                # Skip @ and \
                                # Skip opening brackets & opening quote (\p{Pi}\p{Ps})
                                (?![~`@\\\p{Pi}\p{Ps}])
                                (?=[\p{P}\p{Zs}\p{S}]|$)
                             )
                             """
SLACK_ITALIC_REGEX = r"""
                      # Same as `SLACK_STRIKETHROUGH_REGEX`s. The difference
                      # being, this skips _ instead of ~
                      (
                        (?![_`@\\\p{Pf}\p{Pe}])
                        [\p{P}\p{Zs}\p{S}]|^
                      )
                      (\_)
                          ([^_]+)                    # any character except _
                      (\_)
                      (
                        (?![_`@\\\p{Pi}\p{Ps}])
                        (?=[\p{P}\p{Zs}\p{S}]|$)
                      )
                      """
SLACK_BOLD_REGEX = r"""
                    # Same as `SLACK_STRIKETHROUGH_REGEX`s. The difference
                    # being, this skips * instead of ~
                    (
                        (?![*`@\\\p{Pf}\p{Pe}])
                        [\p{P}\p{Zs}\p{S}]|^
                    )
                    (\*)
                        ([^*]+)                       # any character except *
                    (\*)
                    (
                        (?![*`@\\\p{Pi}\p{Ps}])
                        (?=[\p{P}\p{Zs}\p{S}]|$)
                    )
                    """

FALLBACK_SLACK_USER_FULL_NAME = "Slack user {id}"


def get_user_full_name(user: ZerverFieldsT) -> str:
    if "deleted" in user and user["deleted"] is False:
        return user["real_name"] or user["name"]
    elif user["is_mirror_dummy"]:
        return user["profile"].get("real_name", user["name"])
    else:
        return user["name"]


def get_zulip_mention_for_slack_user(
    slack_user_id: str | None,
    slack_user_shortname: str | None,
    users: list[ZerverFieldsT],
    silent: bool = False,
) -> str | None:
    if slack_user_id:
        for user in users:
            if user["id"] == slack_user_id and (
                slack_user_shortname is None or user["name"] == slack_user_shortname
            ):
                return ("@_**" if silent else "@**") + get_user_full_name(user) + "**"
    return None


def get_user_mentions(
    token: str,
    users: list[ZerverFieldsT],
    slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
) -> tuple[str, int | None]:
    slack_usermention_match = re.search(SLACK_USERMENTION_REGEX, token, re.VERBOSE)
    assert slack_usermention_match is not None
    short_name = slack_usermention_match.group(4)
    slack_id = slack_usermention_match.group(2)
    zulip_mention = get_zulip_mention_for_slack_user(slack_id, short_name, users)
    if zulip_mention is not None:
        token = re.sub(SLACK_USERMENTION_REGEX, zulip_mention, token, flags=re.VERBOSE)
        user_id = slack_user_id_to_zulip_user_id[slack_id]
        return token, user_id
    return token, None


def convert_link_format(text: str) -> tuple[str, bool]:
    """
    1. Converts '<https://foo.com>' to 'https://foo.com'
    2. Converts '<https://foo.com|foo>' to '[foo](https://foo.com)'
    """
    has_link = False
    for match in re.finditer(LINK_REGEX, text, re.VERBOSE):
        slack_url = match.group(0)
        url_parts = slack_url[1:-1].split("|", maxsplit=1)
        # Check if there's a pipe with text after it
        if len(url_parts) == 2:
            converted_url = f"[{url_parts[1]}]({url_parts[0]})"
        else:
            converted_url = url_parts[0]

        has_link = True
        text = text.replace(slack_url, converted_url)
    return text, has_link


def convert_mailto_format(text: str) -> tuple[str, bool]:
    """
    1. Converts '<mailto:[email protected]>' to 'mailto:[email protected]'
    2. Converts '<mailto:[email protected]|[email protected]>' to 'mailto:[email protected]'
    """
    has_link = False
    for match in re.finditer(SLACK_MAILTO_REGEX, text, re.VERBOSE):
        has_link = True
        text = text.replace(match.group(0), match.group(1))
    return text, has_link


# Map italic, bold and strikethrough Markdown
def convert_markdown_syntax(text: str, pattern: str, zulip_keyword: str) -> str:
    """
    Returns:
    1. For strikethrough formatting: This maps Slack's '~strike~' to Zulip's '~~strike~~'
    2. For bold formatting: This maps Slack's '*bold*' to Zulip's '**bold**'
    3. For italic formatting: This maps Slack's '_italic_' to Zulip's '*italic*'
    """

    def replace_slack_format(match: regex.Match[str]) -> str:
        return match.group(1) + zulip_keyword + match.group(3) + zulip_keyword

    return regex.sub(pattern, replace_slack_format, text, flags=re.VERBOSE | re.MULTILINE)


def convert_slack_workspace_mentions(text: str) -> str:
    # Map Slack's '<!everyone>', '<!channel>' and '<!here>'
    # mentions to Zulip's '@**all**' wildcard mention.
    # No regex for these as they can be present anywhere
    # in the sentence.
    text = text.replace("<!everyone>", "@**all**")
    text = text.replace("<!channel>", "@**all**")
    text = text.replace("<!here>", "@**all**")
    return text


def convert_slack_formatting(text: str) -> str:
    text = convert_markdown_syntax(text, SLACK_BOLD_REGEX, "**")
    text = convert_markdown_syntax(text, SLACK_STRIKETHROUGH_REGEX, "~~")
    text = convert_markdown_syntax(text, SLACK_ITALIC_REGEX, "*")
    return text


# Markdown mapping
def convert_to_zulip_markdown(
    text: str,
    users: list[ZerverFieldsT],
    added_channels: AddedChannelsT,
    slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
) -> tuple[str, list[int], bool]:
    mentioned_users_id = []
    text = convert_slack_formatting(text)
    text = convert_slack_workspace_mentions(text)

    # Map Slack channel mention: '<#C5Z73A7RA|general>' to '#**general**'
    for cname, ids in added_channels.items():
        cid = ids[0]
        text = text.replace(f"<#{cid}|{cname}>", "#**" + cname + "**")

    tokens = text.split(" ")
    for iterator in range(len(tokens)):
        # Check user mentions and change mention format from
        # '<@slack_id|short_name>' to '@**full_name**'
        if re.findall(SLACK_USERMENTION_REGEX, tokens[iterator], re.VERBOSE):
            tokens[iterator], user_id = get_user_mentions(
                tokens[iterator], users, slack_user_id_to_zulip_user_id
            )
            if user_id is not None:
                mentioned_users_id.append(user_id)

    text = " ".join(tokens)

    # Check and convert link format
    text, has_link = convert_link_format(text)
    # convert `<mailto:[email protected]>` to `mailto:[email protected]`
    text, has_mailto_link = convert_mailto_format(text)

    message_has_link = has_link or has_mailto_link

    return text, mentioned_users_id, message_has_link


class LossyConversionError(Exception):
    pass


def render_block(block: WildValue) -> str:
    """
    Raises `LossyConversionError` if it's a rich_text block type.
    """
    # https://api.slack.com/reference/block-kit/blocks
    supported_types = {
        "context",
        "divider",
        "header",
        "image",
        "section",
    }
    unhandled_types = {
        # `call` is a block type we've observed in the wild in a Slack export,
        # despite not being documented in
        # https://docs.slack.dev/reference/block-kit/blocks/
        # It likes maps to a request for a Slack call. If we can verify that,
        # probably it would be worth replacing with a string indicating a Slack
        # call occurred.
        "call",
        "contact_card",
        "file",
        "table",
        "video",
        # The "actions" block is used to format literal in-message clickable
        # buttons and similar elements, which Zulip currently doesn't support.
        # https://docs.slack.dev/reference/block-kit/blocks/actions-block
        "actions",
        "input",
        "condition",
    }
    known_types = {
        *supported_types,
        *unhandled_types,
        # rich_text is a special case. It probably composes the bulk of a message
        # body since it's responsible for very basic text formatting--such as
        # plain text, bold, italic, etc. Our current Slack text conversion module
        # can't process this block since this is not formatted in Markdown.
        "rich_text",
    }
    block_type = block["type"].tame(check_string_in(known_types))

    if block_type in unhandled_types:
        return ""
    elif block_type == "rich_text":
        raise LossyConversionError
    elif block_type == "context" and block.get("elements"):
        pieces = []
        # Slack renders these pieces left-to-right, packed in as
        # closely as possible.  We just render them above each other,
        # for simplicity.
        for element in block["elements"]:
            element_type = element["type"].tame(check_string_in(["image", "plain_text", "mrkdwn"]))
            if element_type == "image":
                pieces.append(render_block_element(element))
            else:
                pieces.append(element.tame(check_text_block())["text"])
        return "\n\n".join(piece.strip() for piece in pieces if piece.strip() != "")
    elif block_type == "divider":
        return "----"
    elif block_type == "header":
        return "## " + block["text"].tame(check_text_block(plain_text_only=True))["text"]
    elif block_type == "image":
        image_url = block["image_url"].tame(check_url)
        alt_text = block["alt_text"].tame(check_string)
        if "title" in block:
            alt_text = block["title"].tame(check_text_block(plain_text_only=True))["text"]
        return f"[{alt_text}]({image_url})"
    elif block_type == "section":
        pieces = []
        if "text" in block:
            pieces.append(block["text"].tame(check_text_block())["text"])

        if "accessory" in block:
            pieces.append(render_block_element(block["accessory"]))

        if "fields" in block:
            fields = block["fields"].tame(check_list(check_text_block()))
            if len(fields) == 1:
                # Special-case a single field to display a bit more
                # nicely, without extraneous borders and limitations
                # on its contents.
                pieces.append(fields[0]["text"])
            else:
                # It is not possible to have newlines in a table, nor
                # escape the pipes that make it up; replace them with
                # whitespace.
                field_text = [f["text"].replace("\n", " ").replace("|", " ") for f in fields]
                # Because Slack formats this as two columns, but not
                # necessarily a table with a bold header, we emit a
                # blank header row first.
                table = "| | |\n|-|-|\n"
                # Then take the fields two-at-a-time to make the table
                iters = [iter(field_text)] * 2
                for left, right in zip_longest(*iters, fillvalue=""):
                    table += f"| {left} | {right} |\n"
                pieces.append(table)

        return "\n\n".join(piece.strip() for piece in pieces if piece.strip() != "")

    return ""  # nocoverage


class TextField(TypedDict):
    text: str
    type: Literal["plain_text", "mrkdwn"]


def check_text_block(plain_text_only: bool = False) -> Validator[TextField]:
    if plain_text_only:
        type_validator = check_string_in(["plain_text"])
    else:
        type_validator = check_string_in(["plain_text", "mrkdwn"])

    def f(var_name: str, val: object) -> TextField:
        block = check_dict(
            [
                ("type", type_validator),
                ("text", check_string),
            ],
        )(var_name, val)

        return cast(TextField, block)

    return f


def render_block_element(element: WildValue) -> str:
    # https://api.slack.com/reference/block-kit/block-elements
    # Zulip doesn't support interactive elements, so we only render images here
    element_type = element["type"].tame(check_string)
    if element_type == "image":
        try:
            image_url = element["image_url"].tame(check_url)
            alt_text = element["alt_text"].tame(check_string)
            return f"[{alt_text}]({image_url})"
        except ValidationError:
            # Just drop invalid image URLs, rather than drop the whole message.
            return ""
    else:
        # Unsupported
        return ""


def render_attachment(attachment: WildValue) -> str:
    # https://api.slack.com/reference/messaging/attachments
    # Slack recommends the usage of "blocks" even within attachments; the
    # rest of the fields we handle here are legacy fields. These fields are
    # optional and may contain null values.
    pieces = []
    if attachment.get("title"):
        title = attachment["title"].tame(check_string)
        if attachment.get("title_link"):
            try:
                title_link = attachment["title_link"].tame(check_url)
                title_text = f"## [{title}]({title_link})"
            except ValidationError:
                title_text = f"## {title}"
            pieces.append(title_text)
        else:
            pieces.append(f"## {title}")
    if attachment.get("pretext"):
        pieces.append(attachment["pretext"].tame(check_string))
    if attachment.get("text"):
        pieces.append(attachment["text"].tame(check_string))
    if "fields" in attachment:
        fields = []
        for field in attachment["fields"]:
            if "title" in field and "value" in field and field["title"] and field["value"]:
                title = field["title"].tame(check_string)
                value = field["value"].tame(check_string)
                fields.append(f"*{title}*: {value}")
            elif field.get("title"):
                title = field["title"].tame(check_string)
                fields.append(f"*{title}*")
            elif field.get("value"):
                value = field["value"].tame(check_string)
                fields.append(f"{value}")
        pieces.append("\n".join(fields))
    if attachment.get("blocks"):
        for block in attachment["blocks"]:
            # rich_text blocks in "attachments" elements don't have a
            # corresponding raw value in the "text" field, so we can't
            # fallback to the "text" field in this case like we can do
            # with rich_text in message "blocks".
            # TODO: We should use the raw rich_text strings instead.
            with contextlib.suppress(LossyConversionError):  # nocoverage
                pieces.append(render_block(block))
    if image_url_wv := attachment.get("image_url"):
        try:
            image_url = image_url_wv.tame(check_url)
        except ValidationError:  # nocoverage
            image_url = image_url_wv.tame(check_string)
            image_url = requote_uri(image_url)
        pieces.append(f"[]({image_url})")
    if attachment.get("footer"):
        pieces.append(attachment["footer"].tame(check_string))
    if attachment.get("ts"):
        try:
            time = attachment["ts"].tame(check_int)
        except ValidationError as e:  # nocoverage
            # In some cases Slack has the ts as a string with a float
            # number. The reason is unknown, but we've observed it
            # in the wild several times.
            ts = attachment["ts"].tame(check_string)
            try:
                ts_float = float(ts)
            except ValueError:
                raise e

            time = int(ts_float)
        pieces.append(datetime_to_global_time(datetime.fromtimestamp(time, timezone.utc)))

    return "\n\n".join(piece.strip() for piece in pieces if piece.strip() != "")


def replace_links(text: str) -> str:
    text, _ = convert_link_format(text)
    text, _ = convert_mailto_format(text)
    return text


def process_slack_block_and_attachment(message: WildValue) -> str:
    pieces: list[str] = []
    raw_text = message["text"].tame(check_string)

    if message.get("blocks"):
        try:
            pieces += map(render_block, message["blocks"])
        except LossyConversionError:
            # render_block doesn't yet handle the "rich_text" block, which is used
            # for very basic text formatting. To avoid message content loss, process
            # the message["text"] field instead if rich_text is used.
            pieces.append(raw_text)

    if set(pieces) in ({""}, set()):
        pieces.append(raw_text)

    # The attachments typically don't have a corresponding raw value in
    # message["text"] and only add to it, so the output is appended unconditionally.
    if message.get("attachments"):
        pieces += map(render_attachment, message["attachments"])
    return "\n".join(piece.strip() for piece in pieces if piece.strip() != "")