LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
from collections.abc import Iterable
from os.path import splitext
from typing import NotRequired, TypedDict, Unpack
import magic
from django.core.exceptions import ValidationError
from django.core.files.uploadedfile import InMemoryUploadedFile
from django.template.defaultfilters import filesizeformat
from django.utils.translation import gettext_lazy as _
from unstract.sdk1.file_storage.constants import FileOperationParams
class FileValidationParam(TypedDict):
allowed_mimetypes: NotRequired[Iterable[str]]
allowed_extensions: NotRequired[Iterable[str]]
max_size: NotRequired[int]
min_size: NotRequired[int]
class FileValidator:
"""Validator for files, checking the size, extension and mimetype.
Initialization parameters:
allowed_extensions: iterable with allowed file extensions
ie. ('txt', 'doc')
allowed_mimetypes: iterable with allowed mimetypes
ie. ('image/png', )
min_size: minimum number of bytes allowed
ie. 100
max_size: maximum number of bytes allowed
ie. 24*1024*1024 for 24 MB
"""
extension_message = _(
"Extension '%(extension)s' not allowed. "
"Allowed extensions are: '%(allowed_extensions)s.'"
)
mime_message = _(
"MIME type '%(mimetype)s' is not valid. "
"Allowed types are: %(allowed_mimetypes)s."
)
min_size_message = _(
"The current file %(size)s, which is too small. "
"The minumum file size is %(allowed_size)s."
)
max_size_message = _(
"The current file %(size)s, which is too large. "
"The maximum file size is %(allowed_size)s."
)
def __init__(self, **kwargs: Unpack[FileValidationParam]) -> None:
self.allowed_extensions: Iterable[str] | None = kwargs.pop(
"allowed_extensions", None
)
self.allowed_mimetypes: Iterable[str] | None = kwargs.pop(
"allowed_mimetypes", None
)
self.min_size: int | None = kwargs.pop("min_size", 0)
self.max_size: int | None = kwargs.pop("max_size", None)
def _check_file_extension(self, file: InMemoryUploadedFile) -> None:
ext = splitext(file.name)[1][1:].lower()
if self.allowed_extensions and ext not in self.allowed_extensions:
message = self.extension_message % {
"extension": ext,
"allowed_extensions": ", ".join(self.allowed_extensions),
}
raise ValidationError(message)
def _check_file_mime_type(self, file: InMemoryUploadedFile) -> None:
# TODO: Need to optimise, istead of reading entire file.
mimetype = magic.from_buffer(
file.read(FileOperationParams.READ_ENTIRE_LENGTH), mime=True
)
file.seek(0) # Reset the file pointer to the start
if self.allowed_mimetypes and mimetype not in self.allowed_mimetypes:
message = self.mime_message % {
"mimetype": mimetype,
"allowed_mimetypes": ", ".join(self.allowed_mimetypes),
}
raise ValidationError(message)
def _check_file_size(self, file: InMemoryUploadedFile) -> None:
filesize = len(file)
if (self.max_size is not None) and (self.min_size is not None):
if self.max_size and filesize > self.max_size:
message = self.max_size_message % {
"size": filesizeformat(filesize),
"allowed_size": filesizeformat(self.max_size),
}
raise ValidationError(message)
elif filesize < self.min_size:
message = self.min_size_message % {
"size": filesizeformat(filesize),
"allowed_size": filesizeformat(self.min_size),
}
raise ValidationError(message)
def __call__(self, value: list[InMemoryUploadedFile]) -> None:
"""Check the extension, content type and file size for each file."""
for file in value:
# Check the extension
self._check_file_extension(file)
# Check the content type
self._check_file_mime_type(file)
# Check the file size
self._check_file_size(file)