Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
uploads: Tolerate filenames with an absurdly long extension.
Prakhar Pratyush43041da593d4
diff --git a/zerver/tests/test_upload.py b/zerver/tests/test_upload.pyindex 773c23cc41b..5f3d9e597ca 100644--- a/zerver/tests/test_upload.py+++ b/zerver/tests/test_upload.py@@ -2,7 +2,7 @@ import osimport reimport tempfilefrom datetime import timedelta-from io import StringIO+from io import BytesIO, StringIOfrom unittest import mockfrom unittest.mock import patchfrom urllib.parse import quote@@ -52,6 +52,7 @@ from zerver.lib.upload.s3 import S3UploadBackendfrom zerver.models import Attachment, Message, OnboardingStep, Realm, RealmDomain, UserProfilefrom zerver.models.realms import get_realmfrom zerver.models.users import get_system_bot, get_user_by_delivery_email+from zerver.upload_handler import TEMPORARY_FILE_MAX_EXTENSION_LENGTH, truncate_filename_extensionclass FileUploadTest(UploadSerializeMixin, ZulipTestCase):@@ -2253,6 +2254,51 @@ class EmojiTest(UploadSerializeMixin, ZulipTestCase):resize_mock.assert_called_once()+class TemporaryFileUploadExtensionTests(ZulipTestCase):+ def make_file(self, filename: str) -> BytesIO:+ # We cannot use SimpleUploadedFile here, because Django truncates the+ # name of an UploadedFile to 255 characters, which is precisely the+ # sanitization we want to test happening without.+ uploaded_file = BytesIO(b"zulip!")+ uploaded_file.name = filename+ return uploaded_file++ def test_truncate_filename_extension(self) -> None:+ self.assertEqual(truncate_filename_extension("test.txt"), "test.txt")+ self.assertEqual(truncate_filename_extension("no_extension"), "no_extension")+ self.assertEqual(truncate_filename_extension("tarball.tar.gz"), "tarball.tar.gz")++ # A long extension is truncated, leaving the rest of the name alone.+ truncated = truncate_filename_extension("test." + "a" * 300)+ self.assertEqual(truncated, "test." + "a" * (TEMPORARY_FILE_MAX_EXTENSION_LENGTH - 1))++ # Truncation happens on encoded bytes, and never leaves a character+ # partially encoded. Each snowman takes 3 bytes, so 78 of them plus+ # the "." fit in the 237 bytes available.+ truncated = truncate_filename_extension("test." + "☃" * 200)+ self.assertEqual(truncated, "test." + "☃" * 78)++ def test_upload_file_with_long_extension(self) -> None:+ self.login("hamlet")+ result = self.client_post(+ "/json/user_uploads", {"file": self.make_file("test." + "a" * 300)}+ )+ response_dict = self.assert_json_success(result)+ self.assertEqual(+ "test." + "a" * (TEMPORARY_FILE_MAX_EXTENSION_LENGTH - 1), response_dict["filename"]+ )++ def test_multipart_request_to_nonexistent_endpoint(self) -> None:+ # Our logging middleware parses the request body looking for a+ # "client" parameter before the URL has been resolved, so a filename+ # too long to spool to disk raised an unhandled OSError (500), rather+ # than the 404. See #39874 description for details.+ result = self.client_post(+ "/json/nonexistent_endpoint", {"file": self.make_file("test." + "a" * 300)}+ )+ self.assertEqual(result.status_code, 404)++class SanitizeNameTests(ZulipTestCase):def test_file_name(self) -> None:self.assertEqual(sanitize_name("test.txt"), "test.txt")diff --git a/zerver/upload_handler.py b/zerver/upload_handler.pynew file mode 100644index 00000000000..b783b710121--- /dev/null+++ b/zerver/upload_handler.py@@ -0,0 +1,60 @@+import os+import tempfile++from django.core.files.uploadhandler import TemporaryFileUploadHandler+from typing_extensions import override++# Django spools each uploaded file to a temporary file named+# f"{tempfile.gettempprefix()}{8 random characters}.upload{extension}",+# where the extension is taken verbatim from the client-provided filename.+#+# Since a filename is limited to NAME_MAX (255) bytes, a sufficiently+# long extension makes creating that temporary file fail with+# 'OSError: [Errno 36] File name too long', which Django does not handle.+TEMPORARY_FILE_MAX_EXTENSION_LENGTH = 255 - len(tempfile.gettempprefix()) - 8 - len(".upload")+++def truncate_filename_extension(filename: str) -> str:+ """Shortens filename extensions that are too long to fit in the name of+ the temporary file Django spools an upload to. NAME_MAX limits bytes+ rather than characters, so we truncate the encoded extension and discard+ any character left partially encoded at the end.+ """+ root, extension = os.path.splitext(filename)+ encoded_extension = extension.encode()+ if len(encoded_extension) <= TEMPORARY_FILE_MAX_EXTENSION_LENGTH:+ return filename+ return root + encoded_extension[:TEMPORARY_FILE_MAX_EXTENSION_LENGTH].decode(errors="ignore")+++class ZulipTemporaryFileUploadHandler(TemporaryFileUploadHandler):+ """Django's TemporaryFileUploadHandler, made tolerant of filenames whose+ extension alone exceeds what the filesystem allows in a filename.++ Django does sanitize an over-long filename, in UploadedFile._set_name,+ but only after creating the temporary file whose name embeds the+ extension; shortening the extension is therefore something we have to do+ beforehand. The cap is far longer than any extension in practice.++ This workaround can go away once https://code.djangoproject.com/ticket/37292+ is fixed upstream.+ """++ @override+ def new_file(+ self,+ field_name: str,+ file_name: str,+ content_type: str,+ content_length: int | None,+ charset: str | None = None,+ content_type_extra: dict[str, bytes] | None = None,+ ) -> None:+ super().new_file(+ field_name,+ truncate_filename_extension(file_name),+ content_type,+ content_length,+ charset,+ content_type_extra,+ )diff --git a/zproject/computed_settings.py b/zproject/computed_settings.pyindex 834185e081b..30e261dd4da 100644--- a/zproject/computed_settings.py+++ b/zproject/computed_settings.py@@ -634,6 +634,13 @@ LOCALE_PATHS = (os.path.join(DEPLOY_ROOT, "locale"),)# We want all temporary uploaded files to be stored on disk.FILE_UPLOAD_MAX_MEMORY_SIZE = 0+# Django's default handlers, with the temporary file handler replaced by our+# subclass that does not crash on a filename with a very long extension.+FILE_UPLOAD_HANDLERS = [+ "django.core.files.uploadhandler.MemoryFileUploadHandler",+ "zerver.upload_handler.ZulipTemporaryFileUploadHandler",+]+if DEVELOPMENT or "ZULIP_COLLECTING_STATIC" in os.environ:STATICFILES_DIRS = [os.path.join(DEPLOY_ROOT, "static")]