Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

uploads: Tolerate filenames with an absurdly long extension.

Prakhar Pratyush43041da593d4
diff --git a/zerver/tests/test_upload.py b/zerver/tests/test_upload.py
index 773c23cc41b..5f3d9e597ca 100644
--- a/zerver/tests/test_upload.py
+++ b/zerver/tests/test_upload.py
@@ -2,7 +2,7 @@ import os
import re
import tempfile
from datetime import timedelta
-from io import StringIO
+from io import BytesIO, StringIO
from unittest import mock
from unittest.mock import patch
from urllib.parse import quote
@@ -52,6 +52,7 @@ from zerver.lib.upload.s3 import S3UploadBackend
from zerver.models import Attachment, Message, OnboardingStep, Realm, RealmDomain, UserProfile
from zerver.models.realms import get_realm
from zerver.models.users import get_system_bot, get_user_by_delivery_email
+from zerver.upload_handler import TEMPORARY_FILE_MAX_EXTENSION_LENGTH, truncate_filename_extension
class FileUploadTest(UploadSerializeMixin, ZulipTestCase):
@@ -2253,6 +2254,51 @@ class EmojiTest(UploadSerializeMixin, ZulipTestCase):
resize_mock.assert_called_once()
+class TemporaryFileUploadExtensionTests(ZulipTestCase):
+ def make_file(self, filename: str) -> BytesIO:
+ # We cannot use SimpleUploadedFile here, because Django truncates the
+ # name of an UploadedFile to 255 characters, which is precisely the
+ # sanitization we want to test happening without.
+ uploaded_file = BytesIO(b"zulip!")
+ uploaded_file.name = filename
+ return uploaded_file
+
+ def test_truncate_filename_extension(self) -> None:
+ self.assertEqual(truncate_filename_extension("test.txt"), "test.txt")
+ self.assertEqual(truncate_filename_extension("no_extension"), "no_extension")
+ self.assertEqual(truncate_filename_extension("tarball.tar.gz"), "tarball.tar.gz")
+
+ # A long extension is truncated, leaving the rest of the name alone.
+ truncated = truncate_filename_extension("test." + "a" * 300)
+ self.assertEqual(truncated, "test." + "a" * (TEMPORARY_FILE_MAX_EXTENSION_LENGTH - 1))
+
+ # Truncation happens on encoded bytes, and never leaves a character
+ # partially encoded. Each snowman takes 3 bytes, so 78 of them plus
+ # the "." fit in the 237 bytes available.
+ truncated = truncate_filename_extension("test." + "☃" * 200)
+ self.assertEqual(truncated, "test." + "☃" * 78)
+
+ def test_upload_file_with_long_extension(self) -> None:
+ self.login("hamlet")
+ result = self.client_post(
+ "/json/user_uploads", {"file": self.make_file("test." + "a" * 300)}
+ )
+ response_dict = self.assert_json_success(result)
+ self.assertEqual(
+ "test." + "a" * (TEMPORARY_FILE_MAX_EXTENSION_LENGTH - 1), response_dict["filename"]
+ )
+
+ def test_multipart_request_to_nonexistent_endpoint(self) -> None:
+ # Our logging middleware parses the request body looking for a
+ # "client" parameter before the URL has been resolved, so a filename
+ # too long to spool to disk raised an unhandled OSError (500), rather
+ # than the 404. See #39874 description for details.
+ result = self.client_post(
+ "/json/nonexistent_endpoint", {"file": self.make_file("test." + "a" * 300)}
+ )
+ self.assertEqual(result.status_code, 404)
+
+
class SanitizeNameTests(ZulipTestCase):
def test_file_name(self) -> None:
self.assertEqual(sanitize_name("test.txt"), "test.txt")
diff --git a/zerver/upload_handler.py b/zerver/upload_handler.py
new file mode 100644
index 00000000000..b783b710121
--- /dev/null
+++ b/zerver/upload_handler.py
@@ -0,0 +1,60 @@
+import os
+import tempfile
+
+from django.core.files.uploadhandler import TemporaryFileUploadHandler
+from typing_extensions import override
+
+# Django spools each uploaded file to a temporary file named
+# f"{tempfile.gettempprefix()}{8 random characters}.upload{extension}",
+# where the extension is taken verbatim from the client-provided filename.
+#
+# Since a filename is limited to NAME_MAX (255) bytes, a sufficiently
+# long extension makes creating that temporary file fail with
+# 'OSError: [Errno 36] File name too long', which Django does not handle.
+TEMPORARY_FILE_MAX_EXTENSION_LENGTH = 255 - len(tempfile.gettempprefix()) - 8 - len(".upload")
+
+
+def truncate_filename_extension(filename: str) -> str:
+ """Shortens filename extensions that are too long to fit in the name of
+ the temporary file Django spools an upload to. NAME_MAX limits bytes
+ rather than characters, so we truncate the encoded extension and discard
+ any character left partially encoded at the end.
+ """
+ root, extension = os.path.splitext(filename)
+ encoded_extension = extension.encode()
+ if len(encoded_extension) <= TEMPORARY_FILE_MAX_EXTENSION_LENGTH:
+ return filename
+ return root + encoded_extension[:TEMPORARY_FILE_MAX_EXTENSION_LENGTH].decode(errors="ignore")
+
+
+class ZulipTemporaryFileUploadHandler(TemporaryFileUploadHandler):
+ """Django's TemporaryFileUploadHandler, made tolerant of filenames whose
+ extension alone exceeds what the filesystem allows in a filename.
+
+ Django does sanitize an over-long filename, in UploadedFile._set_name,
+ but only after creating the temporary file whose name embeds the
+ extension; shortening the extension is therefore something we have to do
+ beforehand. The cap is far longer than any extension in practice.
+
+ This workaround can go away once https://code.djangoproject.com/ticket/37292
+ is fixed upstream.
+ """
+
+ @override
+ def new_file(
+ self,
+ field_name: str,
+ file_name: str,
+ content_type: str,
+ content_length: int | None,
+ charset: str | None = None,
+ content_type_extra: dict[str, bytes] | None = None,
+ ) -> None:
+ super().new_file(
+ field_name,
+ truncate_filename_extension(file_name),
+ content_type,
+ content_length,
+ charset,
+ content_type_extra,
+ )
diff --git a/zproject/computed_settings.py b/zproject/computed_settings.py
index 834185e081b..30e261dd4da 100644
--- a/zproject/computed_settings.py
+++ b/zproject/computed_settings.py
@@ -634,6 +634,13 @@ LOCALE_PATHS = (os.path.join(DEPLOY_ROOT, "locale"),)
# We want all temporary uploaded files to be stored on disk.
FILE_UPLOAD_MAX_MEMORY_SIZE = 0
+# Django's default handlers, with the temporary file handler replaced by our
+# subclass that does not crash on a filename with a very long extension.
+FILE_UPLOAD_HANDLERS = [
+ "django.core.files.uploadhandler.MemoryFileUploadHandler",
+ "zerver.upload_handler.ZulipTemporaryFileUploadHandler",
+]
+
if DEVELOPMENT or "ZULIP_COLLECTING_STATIC" in os.environ:
STATICFILES_DIRS = [os.path.join(DEPLOY_ROOT, "static")]