LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix file upload security loopholes (#14)
Gayathrie8df8ab000aa
diff --git a/backend/file_management/constants.py b/backend/file_management/constants.pyindex 82c16e045..e12a31d9b 100644--- a/backend/file_management/constants.py+++ b/backend/file_management/constants.py@@ -3,3 +3,6 @@ class FileInformationKey:FILE_TYPE = "type"FILE_LAST_MODIFIED = "LastModified"FILE_SIZE = "size"+ FILE_UPLOAD_MAX_SIZE = 100 * 1024 * 1024+ FILE_UPLOAD_ALLOWED_EXTENSIONS = ['pdf']+ FILE_UPLOAD_ALLOWED_MIMETYPES = ['application/pdf']\ No newline at end of filediff --git a/backend/file_management/serializer.py b/backend/file_management/serializer.pyindex 5efa0a665..dfd220cfa 100644--- a/backend/file_management/serializer.py+++ b/backend/file_management/serializer.py@@ -1,5 +1,8 @@from rest_framework import serializers+from file_management.constants import FileInformationKey+from utils.FileValidator import FileValidator+class FileInfoSerializer(serializers.Serializer):name = serializers.CharField()@@ -15,13 +18,36 @@ class FileListRequestSerializer(serializers.Serializer):class FileUploadSerializer(serializers.Serializer):- file = serializers.ListField(child=serializers.FileField(), required=True)+ file = serializers.ListField(+ child=serializers.FileField(),+ required=True,+ validators=[+ FileValidator(+ allowed_extensions=FileInformationKey.FILE_UPLOAD_ALLOWED_EXTENSIONS,+ allowed_mimetypes=FileInformationKey.FILE_UPLOAD_ALLOWED_MIMETYPES,+ min_size=0,+ max_size=FileInformationKey.FILE_UPLOAD_MAX_SIZE,+ )+ ],+ )+ # FileExtensionValidator(allowed_extensions=['pdf'])connector_id = serializers.UUIDField()path = serializers.CharField()class FileUploadIdeSerializer(serializers.Serializer):- file = serializers.ListField(child=serializers.FileField(), required=True)+ file = serializers.ListField(+ child=serializers.FileField(),+ required=True,+ validators=[+ FileValidator(+ allowed_extensions=FileInformationKey.FILE_UPLOAD_ALLOWED_EXTENSIONS,+ allowed_mimetypes=FileInformationKey.FILE_UPLOAD_ALLOWED_MIMETYPES,+ min_size=0,+ max_size=FileInformationKey.FILE_UPLOAD_MAX_SIZE,+ )+ ],+ )class FileInfoIdeSerializer(serializers.Serializer):diff --git a/backend/utils/FileValidator.py b/backend/utils/FileValidator.pynew file mode 100644index 000000000..62bbf22a5--- /dev/null+++ b/backend/utils/FileValidator.py@@ -0,0 +1,83 @@+import magic+from os.path import splitext++from django.core.exceptions import ValidationError+from django.template.defaultfilters import filesizeformat+from django.utils.translation import gettext_lazy as _+from django.utils.translation import ngettext_lazy+++class FileValidator(object):+ """+ Validator for files, checking the size, extension and mimetype.++ Initialization parameters:+ allowed_extensions: iterable with allowed file extensions+ ie. ('txt', 'doc')+ allowed_mimetypes: iterable with allowed mimetypes+ ie. ('image/png', )+ min_size: minimum number of bytes allowed+ ie. 100+ max_size: maximum number of bytes allowed+ ie. 24*1024*1024 for 24 MB++ """++ extension_message = _("Extension '%(extension)s' not allowed. "+ "Allowed extensions are: '%(allowed_extensions)s.'")+ mime_message = _("MIME type '%(mimetype)s' is not valid. "+ "Allowed types are: %(allowed_mimetypes)s.")+ min_size_message = _('The current file %(size)s, which is too small. '+ 'The minumum file size is %(allowed_size)s.')+ max_size_message = _('The current file %(size)s, which is too large. '+ 'The maximum file size is %(allowed_size)s.')++ def __init__(self, *args, **kwargs):+ self.allowed_extensions = kwargs.pop('allowed_extensions', None)+ self.allowed_mimetypes = kwargs.pop('allowed_mimetypes', None)+ self.min_size = kwargs.pop('min_size', 0)+ self.max_size = kwargs.pop('max_size', None)++ def __call__(self, value):+ """+ Check the extension, content type and file size for each file+ """+ for file in value:+ # Check the extension+ ext = splitext(file.name)[1][1:].lower()+ if self.allowed_extensions and not ext in self.allowed_extensions:+ message = self.extension_message % {+ 'extension' : ext,+ 'allowed_extensions': ', '.join(self.allowed_extensions)+ }++ raise ValidationError(message)++ # Check the content type+ mimetype = magic.from_buffer(file.read(2048), mime=True)+ if (self.allowed_mimetypes and+ not mimetype in self.allowed_mimetypes):+ message = self.mime_message % {+ 'mimetype': mimetype,+ 'allowed_mimetypes': ', '.join(self.allowed_mimetypes)+ }++ raise ValidationError(message)++ # Check the file size+ filesize = len(file)+ if self.max_size and filesize > self.max_size:+ message = self.max_size_message % {+ 'size': filesizeformat(filesize),+ 'allowed_size': filesizeformat(self.max_size)+ }++ raise ValidationError(message)++ elif filesize < self.min_size:+ message = self.min_size_message % {+ 'size': filesizeformat(filesize),+ 'allowed_size': filesizeformat(self.min_size)+ }++ raise ValidationError(message)\ No newline at end of file