Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

13.5 KB
Raw Blame
from typing import Annotated, Any

import orjson
from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.db import transaction
from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
from django.utils.http import content_disposition_header
from django.utils.translation import gettext as _
from django.views.decorators.csrf import csrf_exempt
from pydantic import BaseModel, ConfigDict, Field
from pydantic.alias_generators import to_pascal

from confirmation.models import Confirmation, ConfirmationKeyError, get_object_from_key
from zerver.decorator import get_basic_credentials, validate_api_key
from zerver.lib.exceptions import AccessDeniedError, JsonableError
from zerver.lib.mime_types import INLINE_MIME_TYPES, bare_content_type, guess_type
from zerver.lib.rate_limiter import is_local_addr
from zerver.lib.typed_endpoint import JsonBodyPayload, typed_endpoint
from zerver.lib.upload import (
    RealmUploadQuotaError,
    attachment_source,
    check_upload_within_quota,
    create_attachment,
    delete_message_attachment,
    generate_message_upload_path,
    get_upload_backend,
    maybe_add_charset,
    needs_charset_detection,
    sanitize_name,
)
from zerver.models import ArchivedAttachment, Attachment, PreregistrationRealm, Realm, UserProfile


# See https://tus.github.io/tusd/advanced-topics/hooks/ for the spec
# for these.
class TusUpload(BaseModel):
    model_config = ConfigDict(populate_by_name=True, alias_generator=to_pascal)
    id: Annotated[str, Field(alias="ID")]
    size: int | None
    size_is_deferred: bool
    offset: int
    meta_data: dict[str, str]
    is_partial: bool
    is_final: bool
    partial_uploads: list[str] | None
    storage: dict[str, str] | None


class TusHTTPRequest(BaseModel):
    model_config = ConfigDict(populate_by_name=True, alias_generator=to_pascal)
    method: str
    uri: Annotated[str, Field(alias="URI")]
    remote_addr: str
    header: dict[str, list[str]]


class TusEvent(BaseModel):
    model_config = ConfigDict(populate_by_name=True, alias_generator=to_pascal)
    upload: TusUpload
    http_request: Annotated[TusHTTPRequest, Field(alias="HTTPRequest")]


class TusHook(BaseModel):
    model_config = ConfigDict(populate_by_name=True, alias_generator=to_pascal)
    type: str
    event: TusEvent


# Note that we do not raise JsonableError in these views
# because our client is not a consumer of the Zulip API -- it's tusd,
# which has its own ideas of what error responses look like.
def tusd_json_response(data: dict[str, Any]) -> HttpResponse:
    return HttpResponse(
        content=orjson.dumps(data, option=orjson.OPT_APPEND_NEWLINE),
        content_type="application/json",
        status=200,
    )


def reject_upload(message: str, status_code: int) -> HttpResponse:
    # Due to https://github.com/transloadit/uppy/issues/5460, uppy
    # will retry responses with a statuscode of exactly 400, so we
    # return 4xx status codes which are more specific to trigger an
    # immediate rejection.
    return tusd_json_response(
        {
            "HttpResponse": {
                "StatusCode": status_code,
                "Body": orjson.dumps({"message": message}).decode(),
                "Header": {
                    "Content-Type": "application/json",
                },
            },
            "RejectUpload": True,
        }
    )


def handle_upload_pre_create_hook(
    request: HttpRequest, user_profile: UserProfile, data: TusUpload
) -> HttpResponse:
    if data.size_is_deferred or data.size is None:
        return reject_upload("SizeIsDeferred is not supported", 411)

    max_file_upload_size_mebibytes = user_profile.realm.get_max_file_upload_size_mebibytes()
    if data.size > max_file_upload_size_mebibytes * 1024 * 1024:
        if user_profile.realm.plan_type != Realm.PLAN_TYPE_SELF_HOSTED:
            return reject_upload(
                _(
                    "File is larger than the maximum upload size ({max_size} MiB) allowed by your organization's plan."
                ).format(
                    max_size=max_file_upload_size_mebibytes,
                ),
                413,
            )
        else:
            return reject_upload(
                _(
                    "File is larger than this server's configured maximum upload size ({max_size} MiB)."
                ).format(
                    max_size=max_file_upload_size_mebibytes,
                ),
                413,
            )

    try:
        check_upload_within_quota(user_profile.realm, data.size)
    except RealmUploadQuotaError as e:
        return reject_upload(str(e), 413)

    # Determine the path_id to store it at
    file_name = sanitize_name(data.meta_data.get("filename", ""), strict=True)
    path_id = generate_message_upload_path(str(user_profile.realm_id), file_name)
    return tusd_json_response({"ChangeFileInfo": {"ID": path_id}})


def handle_upload_pre_finish_hook(
    request: HttpRequest, user_profile: UserProfile, data: TusUpload
) -> HttpResponse:
    # With an S3 backend, the filename we passed in pre_create's
    # data.id has a randomly-generated "multipart-id" appended with a
    # `+`.  Our path_ids cannot contain `+`, so we strip any suffix
    # starting with `+`.
    path_id = data.id.partition("+")[0]

    tus_metadata = data.meta_data
    filename = tus_metadata.get("filename", "")

    # We want to store as the filename a version that clients are
    # likely to be able to accept via "Save as..."
    if filename in {"", ".", ".."}:
        filename = "uploaded-file"

    content_type = tus_metadata.get("filetype")
    if not content_type:
        content_type = guess_type(filename)[0]
        if content_type is None:
            content_type = "application/octet-stream"

    # Only open the object at all when charset detection actually needs to
    # inspect its bytes. With the S3 backend, opening it starts a
    # GetObject read that we'd otherwise hold open across the self-copy
    # below. AWS S3 doesn't mind, but S3-compatible backends that lock per
    # object (e.g. MinIO) hold a read lock for the life of an open
    # GetObject, so the copy would block until that idle reader is torn
    # down -- tens of seconds later.
    if needs_charset_detection(content_type):
        content_type = maybe_add_charset(content_type, attachment_source(path_id))

    if settings.LOCAL_UPLOADS_DIR is None:
        # We "copy" the file to itself to update the Content-Type,
        # Content-Disposition, and storage class of the data.  This
        # parallels the work from upload_content_to_s3 in
        # zerver.lib.uploads.s3
        s3_metadata = {
            "user_profile_id": str(user_profile.id),
            "realm_id": str(user_profile.realm_id),
        }

        is_attachment = bare_content_type(content_type) not in INLINE_MIME_TYPES
        content_disposition = content_disposition_header(is_attachment, filename) or "inline"

        from zerver.lib.upload.s3 import S3UploadBackend

        upload_backend = get_upload_backend()
        assert isinstance(upload_backend, S3UploadBackend)
        key = upload_backend.uploads_bucket.Object(path_id)
        key.copy_from(
            ContentType=content_type,
            ContentDisposition=content_disposition,
            CopySource={"Bucket": settings.S3_AUTH_UPLOADS_BUCKET, "Key": path_id},
            Metadata=s3_metadata,
            MetadataDirective="REPLACE",
            StorageClass=settings.S3_UPLOADS_STORAGE_CLASS,
        )

        # https://tus.github.io/tusd/storage-backends/overview/#storage-format
        # tusd also creates a .info file next to the upload, which
        # must be preserved for HEAD requests (to check for upload
        # state) to work.  These files are inaccessible via Zulip, and
        # small enough to not pose any notable storage use; but we
        # should store them with the right StorageClass.
        if settings.S3_UPLOADS_STORAGE_CLASS != "STANDARD":
            info_key = upload_backend.uploads_bucket.Object(path_id + ".info")
            info_key.copy_from(
                CopySource={"Bucket": settings.S3_AUTH_UPLOADS_BUCKET, "Key": path_id + ".info"},
                MetadataDirective="COPY",
                StorageClass=settings.S3_UPLOADS_STORAGE_CLASS,
            )

    # Open a fresh source for create_attachment/maybe_thumbnail to
    # read from, now that the self-copy above (if any) has finished.
    file_data = attachment_source(path_id)

    with transaction.atomic(durable=True):
        create_attachment(
            filename,
            path_id,
            content_type,
            file_data,
            user_profile,
            user_profile.realm,
        )

    path = "/user_uploads/" + path_id
    return tusd_json_response(
        {
            "HttpResponse": {
                "StatusCode": 200,
                "Body": orjson.dumps({"url": path, "filename": filename}).decode(),
                "Header": {
                    "Content-Type": "application/json",
                },
            },
        }
    )


def handle_upload_pre_terminate_hook(
    request: HttpRequest, user_profile: UserProfile, data: TusUpload
) -> HttpResponse:
    path_id = data.id.partition("+")[0]

    if (
        Attachment.objects.filter(path_id=path_id).exists()
        or ArchivedAttachment.objects.filter(path_id=path_id).exists()
    ):
        # Once we have it in our Attachments table (i.e. the
        # pre-upload-finished hook has run), it is ours to manage and
        # we no longer accept terminations.
        return tusd_json_response({"RejectTermination": True})
    return tusd_json_response({})


def authenticate_user(request: HttpRequest) -> UserProfile | AnonymousUser:
    # This acts like the authenticated_rest_api_view wrapper, while
    # allowing fallback to session-based request.user
    if "Authorization" in request.headers:
        try:
            role, api_key = get_basic_credentials(request)
            return validate_api_key(
                request,
                role,
                api_key,
            )

        except JsonableError:
            pass

    # If that failed, fall back to session auth
    return request.user


def handle_preregistration_pre_create_hook(
    request: HttpRequest, preregistration_realm: PreregistrationRealm, data: TusUpload
) -> HttpResponse:
    if data.size_is_deferred or data.size is None:
        return reject_upload("SizeIsDeferred is not supported", 411)

    if settings.MAX_WEB_DATA_IMPORT_SIZE_MB is not None:
        max_upload_size = settings.MAX_WEB_DATA_IMPORT_SIZE_MB * 1024 * 1024  # 1G
        if data.size > max_upload_size:
            return reject_upload(
                _(
                    "Uploaded file exceeds the maximum file size for imports ({max_file_size} MiB)."
                ).format(max_file_size=settings.MAX_WEB_DATA_IMPORT_SIZE_MB),
                413,
            )

    filename = f"import/{preregistration_realm.id}/slack.zip"

    # Delete any existing upload, so tusd doesn't declare that there's nothing
    # to do. This also has the nice benefit of deleting the previous upload.
    delete_message_attachment(filename)

    return tusd_json_response({"ChangeFileInfo": {"ID": filename}})


def handle_preregistration_pre_finish_hook(
    request: HttpRequest, preregistration_realm: PreregistrationRealm, data: TusUpload
) -> HttpResponse:
    # Save the filename to display the uploaded file to user. We need to store it in
    # the database so that is available even after a refresh.
    filename = data.meta_data["filename"]
    preregistration_realm.data_import_metadata["uploaded_import_file_name"] = filename
    # The next import attempt will report its own errors.
    preregistration_realm.data_import_metadata.pop("invalid_file_error_message", None)
    preregistration_realm.data_import_metadata.pop("invalid_user_emails", None)
    preregistration_realm.save(update_fields=["data_import_metadata"])
    return tusd_json_response({})


@csrf_exempt
@typed_endpoint
def handle_tusd_hook(
    request: HttpRequest,
    *,
    payload: JsonBodyPayload[TusHook],
) -> HttpResponse:
    # Make sure this came from localhost
    if not is_local_addr(request.META["REMOTE_ADDR"]):
        raise AccessDeniedError

    hook_name = payload.type
    maybe_user = authenticate_user(request)
    if maybe_user.is_authenticated:
        # Authenticated requests are file upload requests
        if hook_name == "pre-create":
            return handle_upload_pre_create_hook(request, maybe_user, payload.event.upload)
        elif hook_name == "pre-finish":
            return handle_upload_pre_finish_hook(request, maybe_user, payload.event.upload)
        elif hook_name == "pre-terminate":
            return handle_upload_pre_terminate_hook(request, maybe_user, payload.event.upload)
        else:
            return HttpResponseNotFound()

    # Check if unauthenticated requests are for realm creation
    key = payload.event.upload.meta_data.get("key")
    if key is None:
        return reject_upload("Unauthenticated upload", 401)
    try:
        prereg_object = get_object_from_key(
            key, [Confirmation.NEW_REALM_USER_REGISTRATION], mark_object_used=False
        )
    except ConfirmationKeyError:
        return reject_upload("Unauthenticated upload", 401)

    assert isinstance(prereg_object, PreregistrationRealm)
    assert prereg_object.created_realm is None

    if hook_name == "pre-create":
        return handle_preregistration_pre_create_hook(request, prereg_object, payload.event.upload)
    elif hook_name == "pre-finish":
        return handle_preregistration_pre_finish_hook(request, prereg_object, payload.event.upload)
    else:  # nocoverage
        return HttpResponseNotFound()