LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3393 [FEAT] Add SanitizedSerializerMixin foundation for input validation (#1965)
Chandrasekharan M607eed2c8e4b
diff --git a/backend/utils/input_sanitizer.py b/backend/utils/input_sanitizer.pyindex 227dbc95c..fb5f3f938 100644--- a/backend/utils/input_sanitizer.py+++ b/backend/utils/input_sanitizer.py@@ -1,16 +1,28 @@+import loggingimport re+from typing import NoReturnfrom rest_framework.serializers import ValidationError-# Pattern to detect HTML/script tags (closed tags and unclosed tags starting with a letter)-# The second alternative catches unclosed tags like "<script" or "<img src=x" that could-# be completed by adjacent content in non-React rendering contexts (emails, PDFs, logs)-HTML_TAG_PATTERN = re.compile(r"<[^>]*>|<[a-zA-Z/!]")-# Pattern to detect dangerous URI protocols: javascript:, vbscript:, and data: URIs.-# data: URIs are only matched when followed by a MIME type (word/word) to avoid-# false positives on ordinary English text like "Input data: JSON format".+logger = logging.getLogger(__name__)++# Only tags that can execute script or load remote content are rejected. A blanket+# "any angle bracket" rule cannot be used here: these validators run on free-form+# text (prompts, chat questions, descriptions) where "qty <threshold" and+# "the <invoice_no> field" are ordinary content, not markup.+_SCRIPTABLE_TAGS = (+ "script|iframe|object|embed|svg|math|link|meta|base|form|style|"+ "applet|frame|frameset|template"+)+HTML_TAG_PATTERN = re.compile(rf"<\s*/?\s*({_SCRIPTABLE_TAGS})\b", re.IGNORECASE)+# Pattern to detect dangerous URI protocols: javascript:, vbscript:, and the data:+# MIME types that can carry markup or script. Inert payloads such as+# "data:image/png;base64,..." are left alone, as is prose like "Input data: JSON".JS_PROTOCOL_PATTERN = re.compile(- r"(?:javascript|vbscript)\s*:|data\s*:\s*\w+/\w+", re.IGNORECASE+ r"(?:javascript|vbscript)\s*:|"+ r"data\s*:\s*(?:text/html|image/svg\+xml|"+ r"application/(?:xhtml\+xml|x?-?javascript|ecmascript))",+ re.IGNORECASE,)# Pattern to detect event handlers using a vetted list of DOM event names.# This avoids false positives on benign words like "connection=", "onboarding=", etc.@@ -26,14 +38,34 @@ _DOM_EVENTS = (EVENT_HANDLER_PATTERN = re.compile(rf"\bon({_DOM_EVENTS})\s*=", re.IGNORECASE)+def _reject(field_name: str, reason: str, message: str) -> NoReturn:+ logger.warning(+ "input_validation_rejected",+ extra={"field": field_name, "reason": reason},+ )+ raise ValidationError(message)++def validate_no_html_tags(value: str, field_name: str = "This field") -> str:"""Reject values containing HTML/script tags."""if HTML_TAG_PATTERN.search(value):- raise ValidationError(f"{field_name} must not contain HTML or script tags.")+ _reject(+ field_name,+ "html_tag",+ f"{field_name} must not contain HTML or script tags.",+ )if JS_PROTOCOL_PATTERN.search(value):- raise ValidationError(f"{field_name} must not contain dangerous URI protocols.")+ _reject(+ field_name,+ "js_protocol",+ f"{field_name} must not contain dangerous URI protocols.",+ )if EVENT_HANDLER_PATTERN.search(value):- raise ValidationError(f"{field_name} must not contain event handler attributes.")+ _reject(+ field_name,+ "event_handler",+ f"{field_name} must not contain event handler attributes.",+ )return valuediff --git a/backend/utils/serializer/__init__.py b/backend/utils/serializer/__init__.pynew file mode 100644index 000000000..0f2b61acb--- /dev/null+++ b/backend/utils/serializer/__init__.py@@ -0,0 +1,15 @@+from utils.serializer.integrity_error_mixin import IntegrityErrorMixin+from utils.serializer.sanitization import (+ HyperlinkedModelSerializer,+ ModelSerializer,+ SanitizedSerializerMixin,+ Serializer,+)++__all__ = [+ "HyperlinkedModelSerializer",+ "IntegrityErrorMixin",+ "ModelSerializer",+ "SanitizedSerializerMixin",+ "Serializer",+]diff --git a/backend/utils/serializer/sanitization.py b/backend/utils/serializer/sanitization.pynew file mode 100644index 000000000..3c53ed73d--- /dev/null+++ b/backend/utils/serializer/sanitization.py@@ -0,0 +1,55 @@+"""Sanitized serializer base classes.++`SanitizedSerializerMixin` attaches `validate_no_html_tags` to every writable+`CharField` on the serializer, so new serializers are protected from stored+XSS without per-field wiring. Opt out per-serializer via+`Meta.html_safe_fields = (...)` for fields that legitimately accept HTML-like+content (e.g. prompt text, regex literals).++Use the pre-mixed `ModelSerializer` / `Serializer` / `HyperlinkedModelSerializer`+classes instead of importing from `rest_framework` directly:++ from utils.serializer import ModelSerializer++ class FooSerializer(ModelSerializer):+ class Meta:+ model = Foo+ fields = ["name", "description", "prompt"]+ html_safe_fields = ("prompt",) # opt-out+"""++from functools import partial++from rest_framework import serializers as drf++from utils.input_sanitizer import validate_no_html_tags+++class SanitizedSerializerMixin:+ """Attach `validate_no_html_tags` to every writable CharField."""++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ meta = getattr(self, "Meta", None)+ exempt = set(getattr(meta, "html_safe_fields", ()) or ())+ for name, field in self.fields.items():+ if name in exempt or field.read_only:+ continue+ if isinstance(field, drf.CharField):+ # partial binds the field name at iteration time, avoiding the+ # late-binding closure trap of a bare lambda.+ field.validators.append(partial(validate_no_html_tags, field_name=name))+++class ModelSerializer(SanitizedSerializerMixin, drf.ModelSerializer):+ pass+++class Serializer(SanitizedSerializerMixin, drf.Serializer):+ pass+++class HyperlinkedModelSerializer(+ SanitizedSerializerMixin, drf.HyperlinkedModelSerializer+):+ passdiff --git a/backend/utils/tests/test_input_sanitizer.py b/backend/utils/tests/test_input_sanitizer.pyindex 159690488..5deeab9d0 100644--- a/backend/utils/tests/test_input_sanitizer.py+++ b/backend/utils/tests/test_input_sanitizer.py@@ -25,29 +25,57 @@ class TestValidateNoHtmlTags:with pytest.raises(ValidationError, match="must not contain HTML or script tags"):validate_no_html_tags("<script>alert(1)</script>")- def test_rejects_img_tag(self):+ def test_rejects_img_tag_via_event_handler(self):+ with pytest.raises(+ ValidationError, match="must not contain event handler attributes"+ ):+ validate_no_html_tags("<img src=x onerror=alert(1)>")++ def test_rejects_iframe_tag(self):with pytest.raises(ValidationError, match="must not contain HTML or script tags"):- validate_no_html_tags('<img src=x onerror=alert(1)>')+ validate_no_html_tags("<iframe src=//evil.example></iframe>")- def test_rejects_div_tag(self):+ def test_rejects_svg_tag(self):with pytest.raises(ValidationError, match="must not contain HTML or script tags"):- validate_no_html_tags("<div>content</div>")+ validate_no_html_tags("<svg/onload=alert(1)>")- def test_rejects_self_closing_tag(self):+ def test_rejects_closing_script_tag(self):with pytest.raises(ValidationError, match="must not contain HTML or script tags"):- validate_no_html_tags("<br/>")+ validate_no_html_tags("</script>")++ def test_rejects_script_tag_with_inner_space(self):+ with pytest.raises(ValidationError, match="must not contain HTML or script tags"):+ validate_no_html_tags("< script >alert(1)")def test_rejects_unclosed_script_tag(self):with pytest.raises(ValidationError, match="must not contain HTML or script tags"):validate_no_html_tags("<script")- def test_rejects_unclosed_img_tag(self):- with pytest.raises(ValidationError, match="must not contain HTML or script tags"):- validate_no_html_tags("<img src=x onerror")+ def test_allows_inert_markup(self):+ assert validate_no_html_tags("<div>content</div>") == "<div>content</div>"+ assert validate_no_html_tags("Use <br> to break lines.") == (+ "Use <br> to break lines."+ )def test_allows_less_than_with_number(self):assert validate_no_html_tags("a < 3") == "a < 3"+ def test_allows_comparison_without_space(self):+ assert validate_no_html_tags("rows where qty <threshold") == (+ "rows where qty <threshold"+ )+ assert validate_no_html_tags("if x<y then return 1") == "if x<y then return 1"++ def test_allows_angle_bracket_placeholders(self):+ assert validate_no_html_tags("extract the <invoice_no> field") == (+ "extract the <invoice_no> field"+ )++ def test_allows_currency_range(self):+ assert validate_no_html_tags("total is < 500 USD, tax > 5%") == (+ "total is < 500 USD, tax > 5%"+ )+def test_rejects_javascript_protocol(self):with pytest.raises(ValidationError, match="must not contain dangerous URI protocols"):validate_no_html_tags("javascript:alert(1)")@@ -68,6 +96,10 @@ class TestValidateNoHtmlTags:with pytest.raises(ValidationError, match="must not contain dangerous URI protocols"):validate_no_html_tags("data:application/javascript,code")+ def test_allows_inert_data_uri(self):+ value = "See data:image/png;base64,iVBORw0KGgo="+ assert validate_no_html_tags(value) == value+def test_allows_benign_data_colon_in_text(self):assert validate_no_html_tags("Input data: JSON format") == "Input data: JSON format"@@ -108,6 +140,12 @@ class TestValidateNoHtmlTags:def test_allows_condition_text(self):assert validate_no_html_tags("condition = good") == "condition = good"+ def test_allows_spaced_event_word(self):+ assert validate_no_html_tags("on error = retry") == "on error = retry"++ def test_allows_spaced_event_word_without_padding(self):+ assert validate_no_html_tags("on change=true") == "on change=true"+def test_rejects_event_handler_after_semicolon(self):with pytest.raises(ValidationError, match="must not contain event handler attributes"diff --git a/backend/utils/tests/test_sanitized_serializer_mixin.py b/backend/utils/tests/test_sanitized_serializer_mixin.pynew file mode 100644index 000000000..18c537a72--- /dev/null+++ b/backend/utils/tests/test_sanitized_serializer_mixin.py@@ -0,0 +1,113 @@+import pytest+from rest_framework import serializers as drf+from rest_framework.exceptions import ValidationError++from utils.serializer import (+ HyperlinkedModelSerializer,+ ModelSerializer,+ SanitizedSerializerMixin,+ Serializer,+)+++class PlainSerializer(Serializer):+ name = drf.CharField(max_length=100)+ description = drf.CharField(max_length=500, allow_blank=True)+ count = drf.IntegerField(required=False)+++class WithOptOutSerializer(Serializer):+ name = drf.CharField(max_length=100)+ prompt = drf.CharField()++ class Meta:+ html_safe_fields = ("prompt",)+++class WithReadOnlySerializer(Serializer):+ name = drf.CharField(max_length=100)+ computed = drf.CharField(read_only=True)+++class TestSanitizedSerializerMixin:+ def test_rejects_html_in_writable_charfield(self):+ s = PlainSerializer(data={"name": "<script>alert(1)</script>", "description": ""})+ assert not s.is_valid()+ assert "name" in s.errors++ def test_rejects_html_in_description(self):+ s = PlainSerializer(data={"name": "ok", "description": "<img onerror=x>"})+ assert not s.is_valid()+ assert "description" in s.errors++ def test_clean_input_passes(self):+ s = PlainSerializer(data={"name": "My Workflow", "description": "Plain text."})+ assert s.is_valid(), s.errors++ def test_does_not_touch_non_charfield(self):+ s = PlainSerializer(data={"name": "ok", "description": "", "count": 42})+ assert s.is_valid(), s.errors++ def test_each_field_gets_its_own_validator(self):+ """Default-arg closure capture: the field_name in the error must match the offender."""+ s = PlainSerializer(data={"name": "ok", "description": "<script>"})+ assert not s.is_valid()+ assert "description" in s.errors+ msg = str(s.errors["description"][0])+ assert "description" in msg.lower()++ def test_html_safe_fields_opts_out(self):+ s = WithOptOutSerializer(+ data={"name": "ok", "prompt": "<thinking>step 1</thinking>"}+ )+ assert s.is_valid(), s.errors++ def test_html_safe_fields_does_not_leak_to_other_fields(self):+ s = WithOptOutSerializer(+ data={"name": "<script>", "prompt": "<thinking>step 1</thinking>"}+ )+ assert not s.is_valid()+ assert "name" in s.errors++ def test_read_only_field_is_naturally_exempt(self):+ s = WithReadOnlySerializer(data={"name": "ok"})+ assert s.is_valid(), s.errors++ def test_missing_meta_does_not_break(self):+ class NoMetaSerializer(Serializer):+ name = drf.CharField()++ s = NoMetaSerializer(data={"name": "ok"})+ assert s.is_valid(), s.errors++ def test_meta_without_html_safe_fields_does_not_break(self):+ class MetaWithoutAttrSerializer(Serializer):+ name = drf.CharField()++ class Meta:+ pass++ s = MetaWithoutAttrSerializer(data={"name": "<script>"})+ assert not s.is_valid()+ assert "name" in s.errors++ def test_pre_mixed_classes_inherit_mixin(self):+ assert issubclass(ModelSerializer, SanitizedSerializerMixin)+ assert issubclass(Serializer, SanitizedSerializerMixin)+ assert issubclass(HyperlinkedModelSerializer, SanitizedSerializerMixin)++ def test_rejects_javascript_protocol(self):+ s = PlainSerializer(data={"name": "javascript:alert(1)", "description": ""})+ assert not s.is_valid()+ assert "name" in s.errors++ def test_rejects_event_handler(self):+ s = PlainSerializer(data={"name": "onclick=alert(1)", "description": ""})+ assert not s.is_valid()+ assert "name" in s.errors++ def test_validation_raises_validation_error_type(self):+ """End-to-end: raise_exception path surfaces a DRF ValidationError."""+ s = PlainSerializer(data={"name": "<script>", "description": ""})+ with pytest.raises(ValidationError):+ s.is_valid(raise_exception=True)