LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix/file upload security (#31)
Gayathrib8ef3e4b9dc2
diff --git a/backend/file_management/serializer.py b/backend/file_management/serializer.pyindex dfd220cfa..48f99ad3d 100644--- a/backend/file_management/serializer.py+++ b/backend/file_management/serializer.py@@ -1,6 +1,5 @@-from rest_framework import serializers-from file_management.constants import FileInformationKey+from rest_framework import serializersfrom utils.FileValidator import FileValidator@@ -27,7 +26,7 @@ class FileUploadSerializer(serializers.Serializer):allowed_mimetypes=FileInformationKey.FILE_UPLOAD_ALLOWED_MIMETYPES,min_size=0,max_size=FileInformationKey.FILE_UPLOAD_MAX_SIZE,- )+ ) # type: ignore],)# FileExtensionValidator(allowed_extensions=['pdf'])@@ -45,7 +44,7 @@ class FileUploadIdeSerializer(serializers.Serializer):allowed_mimetypes=FileInformationKey.FILE_UPLOAD_ALLOWED_MIMETYPES,min_size=0,max_size=FileInformationKey.FILE_UPLOAD_MAX_SIZE,- )+ ) # type: ignore],)diff --git a/backend/utils/FileValidator.py b/backend/utils/FileValidator.pyindex 62bbf22a5..70af1fe83 100644--- a/backend/utils/FileValidator.py+++ b/backend/utils/FileValidator.py@@ -1,15 +1,13 @@-import magicfrom os.path import splitext+import magicfrom django.core.exceptions import ValidationErrorfrom django.template.defaultfilters import filesizeformatfrom django.utils.translation import gettext_lazy as _-from django.utils.translation import ngettext_lazy-class FileValidator(object):- """- Validator for files, checking the size, extension and mimetype.+class FileValidator:+ """Validator for files, checking the size, extension and mimetype.Initialization parameters:allowed_extensions: iterable with allowed file extensions@@ -20,46 +18,55 @@ class FileValidator(object):ie. 100max_size: maximum number of bytes allowedie. 24*1024*1024 for 24 MB-"""- extension_message = _("Extension '%(extension)s' not allowed. "- "Allowed extensions are: '%(allowed_extensions)s.'")- mime_message = _("MIME type '%(mimetype)s' is not valid. "- "Allowed types are: %(allowed_mimetypes)s.")- min_size_message = _('The current file %(size)s, which is too small. '- 'The minumum file size is %(allowed_size)s.')- max_size_message = _('The current file %(size)s, which is too large. '- 'The maximum file size is %(allowed_size)s.')+ extension_message = _(+ "Extension '%(extension)s' not allowed. "+ "Allowed extensions are: '%(allowed_extensions)s.'"+ )+ mime_message = _(+ "MIME type '%(mimetype)s' is not valid. "+ "Allowed types are: %(allowed_mimetypes)s."+ )+ min_size_message = _(+ "The current file %(size)s, which is too small. "+ "The minumum file size is %(allowed_size)s."+ )+ max_size_message = _(+ "The current file %(size)s, which is too large. "+ "The maximum file size is %(allowed_size)s."+ )- def __init__(self, *args, **kwargs):- self.allowed_extensions = kwargs.pop('allowed_extensions', None)- self.allowed_mimetypes = kwargs.pop('allowed_mimetypes', None)- self.min_size = kwargs.pop('min_size', 0)- self.max_size = kwargs.pop('max_size', None)+ def __init__(self, *args, **kwargs): # type: ignore+ self.allowed_extensions = kwargs.pop("allowed_extensions", None)+ self.allowed_mimetypes = kwargs.pop("allowed_mimetypes", None)+ self.min_size = kwargs.pop("min_size", 0)+ self.max_size = kwargs.pop("max_size", None)- def __call__(self, value):- """- Check the extension, content type and file size for each file- """+ def __call__(self, value): # type: ignore+ """Check the extension, content type and file size for each file."""for file in value:# Check the extensionext = splitext(file.name)[1][1:].lower()- if self.allowed_extensions and not ext in self.allowed_extensions:+ if self.allowed_extensions and ext not in self.allowed_extensions:message = self.extension_message % {- 'extension' : ext,- 'allowed_extensions': ', '.join(self.allowed_extensions)+ "extension": ext,+ "allowed_extensions": ", ".join(self.allowed_extensions),}raise ValidationError(message)# Check the content typemimetype = magic.from_buffer(file.read(2048), mime=True)- if (self.allowed_mimetypes and- not mimetype in self.allowed_mimetypes):+ file.seek(0) # Reset the file pointer to the start++ if (+ self.allowed_mimetypes+ and mimetype not in self.allowed_mimetypes+ ):message = self.mime_message % {- 'mimetype': mimetype,- 'allowed_mimetypes': ', '.join(self.allowed_mimetypes)+ "mimetype": mimetype,+ "allowed_mimetypes": ", ".join(self.allowed_mimetypes),}raise ValidationError(message)@@ -68,16 +75,16 @@ class FileValidator(object):filesize = len(file)if self.max_size and filesize > self.max_size:message = self.max_size_message % {- 'size': filesizeformat(filesize),- 'allowed_size': filesizeformat(self.max_size)+ "size": filesizeformat(filesize),+ "allowed_size": filesizeformat(self.max_size),}raise ValidationError(message)elif filesize < self.min_size:message = self.min_size_message % {- 'size': filesizeformat(filesize),- 'allowed_size': filesizeformat(self.min_size)+ "size": filesizeformat(filesize),+ "allowed_size": filesizeformat(self.min_size),}- raise ValidationError(message)\ No newline at end of file+ raise ValidationError(message)