Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3393 [FEAT] Add SanitizedSerializerMixin foundation for input validation (#1965)

Chandrasekharan M607eed2c8e4b
diff --git a/backend/utils/input_sanitizer.py b/backend/utils/input_sanitizer.py
index 227dbc95c..fb5f3f938 100644
--- a/backend/utils/input_sanitizer.py
+++ b/backend/utils/input_sanitizer.py
@@ -1,16 +1,28 @@
+import logging
import re
+from typing import NoReturn
from rest_framework.serializers import ValidationError
-# Pattern to detect HTML/script tags (closed tags and unclosed tags starting with a letter)
-# The second alternative catches unclosed tags like "<script" or "<img src=x" that could
-# be completed by adjacent content in non-React rendering contexts (emails, PDFs, logs)
-HTML_TAG_PATTERN = re.compile(r"<[^>]*>|<[a-zA-Z/!]")
-# Pattern to detect dangerous URI protocols: javascript:, vbscript:, and data: URIs.
-# data: URIs are only matched when followed by a MIME type (word/word) to avoid
-# false positives on ordinary English text like "Input data: JSON format".
+logger = logging.getLogger(__name__)
+
+# Only tags that can execute script or load remote content are rejected. A blanket
+# "any angle bracket" rule cannot be used here: these validators run on free-form
+# text (prompts, chat questions, descriptions) where "qty <threshold" and
+# "the <invoice_no> field" are ordinary content, not markup.
+_SCRIPTABLE_TAGS = (
+ "script|iframe|object|embed|svg|math|link|meta|base|form|style|"
+ "applet|frame|frameset|template"
+)
+HTML_TAG_PATTERN = re.compile(rf"<\s*/?\s*({_SCRIPTABLE_TAGS})\b", re.IGNORECASE)
+# Pattern to detect dangerous URI protocols: javascript:, vbscript:, and the data:
+# MIME types that can carry markup or script. Inert payloads such as
+# "data:image/png;base64,..." are left alone, as is prose like "Input data: JSON".
JS_PROTOCOL_PATTERN = re.compile(
- r"(?:javascript|vbscript)\s*:|data\s*:\s*\w+/\w+", re.IGNORECASE
+ r"(?:javascript|vbscript)\s*:|"
+ r"data\s*:\s*(?:text/html|image/svg\+xml|"
+ r"application/(?:xhtml\+xml|x?-?javascript|ecmascript))",
+ re.IGNORECASE,
)
# Pattern to detect event handlers using a vetted list of DOM event names.
# This avoids false positives on benign words like "connection=", "onboarding=", etc.
@@ -26,14 +38,34 @@ _DOM_EVENTS = (
EVENT_HANDLER_PATTERN = re.compile(rf"\bon({_DOM_EVENTS})\s*=", re.IGNORECASE)
+def _reject(field_name: str, reason: str, message: str) -> NoReturn:
+ logger.warning(
+ "input_validation_rejected",
+ extra={"field": field_name, "reason": reason},
+ )
+ raise ValidationError(message)
+
+
def validate_no_html_tags(value: str, field_name: str = "This field") -> str:
"""Reject values containing HTML/script tags."""
if HTML_TAG_PATTERN.search(value):
- raise ValidationError(f"{field_name} must not contain HTML or script tags.")
+ _reject(
+ field_name,
+ "html_tag",
+ f"{field_name} must not contain HTML or script tags.",
+ )
if JS_PROTOCOL_PATTERN.search(value):
- raise ValidationError(f"{field_name} must not contain dangerous URI protocols.")
+ _reject(
+ field_name,
+ "js_protocol",
+ f"{field_name} must not contain dangerous URI protocols.",
+ )
if EVENT_HANDLER_PATTERN.search(value):
- raise ValidationError(f"{field_name} must not contain event handler attributes.")
+ _reject(
+ field_name,
+ "event_handler",
+ f"{field_name} must not contain event handler attributes.",
+ )
return value
diff --git a/backend/utils/serializer/__init__.py b/backend/utils/serializer/__init__.py
new file mode 100644
index 000000000..0f2b61acb
--- /dev/null
+++ b/backend/utils/serializer/__init__.py
@@ -0,0 +1,15 @@
+from utils.serializer.integrity_error_mixin import IntegrityErrorMixin
+from utils.serializer.sanitization import (
+ HyperlinkedModelSerializer,
+ ModelSerializer,
+ SanitizedSerializerMixin,
+ Serializer,
+)
+
+__all__ = [
+ "HyperlinkedModelSerializer",
+ "IntegrityErrorMixin",
+ "ModelSerializer",
+ "SanitizedSerializerMixin",
+ "Serializer",
+]
diff --git a/backend/utils/serializer/sanitization.py b/backend/utils/serializer/sanitization.py
new file mode 100644
index 000000000..3c53ed73d
--- /dev/null
+++ b/backend/utils/serializer/sanitization.py
@@ -0,0 +1,55 @@
+"""Sanitized serializer base classes.
+
+`SanitizedSerializerMixin` attaches `validate_no_html_tags` to every writable
+`CharField` on the serializer, so new serializers are protected from stored
+XSS without per-field wiring. Opt out per-serializer via
+`Meta.html_safe_fields = (...)` for fields that legitimately accept HTML-like
+content (e.g. prompt text, regex literals).
+
+Use the pre-mixed `ModelSerializer` / `Serializer` / `HyperlinkedModelSerializer`
+classes instead of importing from `rest_framework` directly:
+
+ from utils.serializer import ModelSerializer
+
+ class FooSerializer(ModelSerializer):
+ class Meta:
+ model = Foo
+ fields = ["name", "description", "prompt"]
+ html_safe_fields = ("prompt",) # opt-out
+"""
+
+from functools import partial
+
+from rest_framework import serializers as drf
+
+from utils.input_sanitizer import validate_no_html_tags
+
+
+class SanitizedSerializerMixin:
+ """Attach `validate_no_html_tags` to every writable CharField."""
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ meta = getattr(self, "Meta", None)
+ exempt = set(getattr(meta, "html_safe_fields", ()) or ())
+ for name, field in self.fields.items():
+ if name in exempt or field.read_only:
+ continue
+ if isinstance(field, drf.CharField):
+ # partial binds the field name at iteration time, avoiding the
+ # late-binding closure trap of a bare lambda.
+ field.validators.append(partial(validate_no_html_tags, field_name=name))
+
+
+class ModelSerializer(SanitizedSerializerMixin, drf.ModelSerializer):
+ pass
+
+
+class Serializer(SanitizedSerializerMixin, drf.Serializer):
+ pass
+
+
+class HyperlinkedModelSerializer(
+ SanitizedSerializerMixin, drf.HyperlinkedModelSerializer
+):
+ pass
diff --git a/backend/utils/tests/test_input_sanitizer.py b/backend/utils/tests/test_input_sanitizer.py
index 159690488..5deeab9d0 100644
--- a/backend/utils/tests/test_input_sanitizer.py
+++ b/backend/utils/tests/test_input_sanitizer.py
@@ -25,29 +25,57 @@ class TestValidateNoHtmlTags:
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
validate_no_html_tags("<script>alert(1)</script>")
- def test_rejects_img_tag(self):
+ def test_rejects_img_tag_via_event_handler(self):
+ with pytest.raises(
+ ValidationError, match="must not contain event handler attributes"
+ ):
+ validate_no_html_tags("<img src=x onerror=alert(1)>")
+
+ def test_rejects_iframe_tag(self):
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
- validate_no_html_tags('<img src=x onerror=alert(1)>')
+ validate_no_html_tags("<iframe src=//evil.example></iframe>")
- def test_rejects_div_tag(self):
+ def test_rejects_svg_tag(self):
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
- validate_no_html_tags("<div>content</div>")
+ validate_no_html_tags("<svg/onload=alert(1)>")
- def test_rejects_self_closing_tag(self):
+ def test_rejects_closing_script_tag(self):
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
- validate_no_html_tags("<br/>")
+ validate_no_html_tags("</script>")
+
+ def test_rejects_script_tag_with_inner_space(self):
+ with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
+ validate_no_html_tags("< script >alert(1)")
def test_rejects_unclosed_script_tag(self):
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
validate_no_html_tags("<script")
- def test_rejects_unclosed_img_tag(self):
- with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
- validate_no_html_tags("<img src=x onerror")
+ def test_allows_inert_markup(self):
+ assert validate_no_html_tags("<div>content</div>") == "<div>content</div>"
+ assert validate_no_html_tags("Use <br> to break lines.") == (
+ "Use <br> to break lines."
+ )
def test_allows_less_than_with_number(self):
assert validate_no_html_tags("a < 3") == "a < 3"
+ def test_allows_comparison_without_space(self):
+ assert validate_no_html_tags("rows where qty <threshold") == (
+ "rows where qty <threshold"
+ )
+ assert validate_no_html_tags("if x<y then return 1") == "if x<y then return 1"
+
+ def test_allows_angle_bracket_placeholders(self):
+ assert validate_no_html_tags("extract the <invoice_no> field") == (
+ "extract the <invoice_no> field"
+ )
+
+ def test_allows_currency_range(self):
+ assert validate_no_html_tags("total is < 500 USD, tax > 5%") == (
+ "total is < 500 USD, tax > 5%"
+ )
+
def test_rejects_javascript_protocol(self):
with pytest.raises(ValidationError, match="must not contain dangerous URI protocols"):
validate_no_html_tags("javascript:alert(1)")
@@ -68,6 +96,10 @@ class TestValidateNoHtmlTags:
with pytest.raises(ValidationError, match="must not contain dangerous URI protocols"):
validate_no_html_tags("data:application/javascript,code")
+ def test_allows_inert_data_uri(self):
+ value = "See data:image/png;base64,iVBORw0KGgo="
+ assert validate_no_html_tags(value) == value
+
def test_allows_benign_data_colon_in_text(self):
assert validate_no_html_tags("Input data: JSON format") == "Input data: JSON format"
@@ -108,6 +140,12 @@ class TestValidateNoHtmlTags:
def test_allows_condition_text(self):
assert validate_no_html_tags("condition = good") == "condition = good"
+ def test_allows_spaced_event_word(self):
+ assert validate_no_html_tags("on error = retry") == "on error = retry"
+
+ def test_allows_spaced_event_word_without_padding(self):
+ assert validate_no_html_tags("on change=true") == "on change=true"
+
def test_rejects_event_handler_after_semicolon(self):
with pytest.raises(
ValidationError, match="must not contain event handler attributes"
diff --git a/backend/utils/tests/test_sanitized_serializer_mixin.py b/backend/utils/tests/test_sanitized_serializer_mixin.py
new file mode 100644
index 000000000..18c537a72
--- /dev/null
+++ b/backend/utils/tests/test_sanitized_serializer_mixin.py
@@ -0,0 +1,113 @@
+import pytest
+from rest_framework import serializers as drf
+from rest_framework.exceptions import ValidationError
+
+from utils.serializer import (
+ HyperlinkedModelSerializer,
+ ModelSerializer,
+ SanitizedSerializerMixin,
+ Serializer,
+)
+
+
+class PlainSerializer(Serializer):
+ name = drf.CharField(max_length=100)
+ description = drf.CharField(max_length=500, allow_blank=True)
+ count = drf.IntegerField(required=False)
+
+
+class WithOptOutSerializer(Serializer):
+ name = drf.CharField(max_length=100)
+ prompt = drf.CharField()
+
+ class Meta:
+ html_safe_fields = ("prompt",)
+
+
+class WithReadOnlySerializer(Serializer):
+ name = drf.CharField(max_length=100)
+ computed = drf.CharField(read_only=True)
+
+
+class TestSanitizedSerializerMixin:
+ def test_rejects_html_in_writable_charfield(self):
+ s = PlainSerializer(data={"name": "<script>alert(1)</script>", "description": ""})
+ assert not s.is_valid()
+ assert "name" in s.errors
+
+ def test_rejects_html_in_description(self):
+ s = PlainSerializer(data={"name": "ok", "description": "<img onerror=x>"})
+ assert not s.is_valid()
+ assert "description" in s.errors
+
+ def test_clean_input_passes(self):
+ s = PlainSerializer(data={"name": "My Workflow", "description": "Plain text."})
+ assert s.is_valid(), s.errors
+
+ def test_does_not_touch_non_charfield(self):
+ s = PlainSerializer(data={"name": "ok", "description": "", "count": 42})
+ assert s.is_valid(), s.errors
+
+ def test_each_field_gets_its_own_validator(self):
+ """Default-arg closure capture: the field_name in the error must match the offender."""
+ s = PlainSerializer(data={"name": "ok", "description": "<script>"})
+ assert not s.is_valid()
+ assert "description" in s.errors
+ msg = str(s.errors["description"][0])
+ assert "description" in msg.lower()
+
+ def test_html_safe_fields_opts_out(self):
+ s = WithOptOutSerializer(
+ data={"name": "ok", "prompt": "<thinking>step 1</thinking>"}
+ )
+ assert s.is_valid(), s.errors
+
+ def test_html_safe_fields_does_not_leak_to_other_fields(self):
+ s = WithOptOutSerializer(
+ data={"name": "<script>", "prompt": "<thinking>step 1</thinking>"}
+ )
+ assert not s.is_valid()
+ assert "name" in s.errors
+
+ def test_read_only_field_is_naturally_exempt(self):
+ s = WithReadOnlySerializer(data={"name": "ok"})
+ assert s.is_valid(), s.errors
+
+ def test_missing_meta_does_not_break(self):
+ class NoMetaSerializer(Serializer):
+ name = drf.CharField()
+
+ s = NoMetaSerializer(data={"name": "ok"})
+ assert s.is_valid(), s.errors
+
+ def test_meta_without_html_safe_fields_does_not_break(self):
+ class MetaWithoutAttrSerializer(Serializer):
+ name = drf.CharField()
+
+ class Meta:
+ pass
+
+ s = MetaWithoutAttrSerializer(data={"name": "<script>"})
+ assert not s.is_valid()
+ assert "name" in s.errors
+
+ def test_pre_mixed_classes_inherit_mixin(self):
+ assert issubclass(ModelSerializer, SanitizedSerializerMixin)
+ assert issubclass(Serializer, SanitizedSerializerMixin)
+ assert issubclass(HyperlinkedModelSerializer, SanitizedSerializerMixin)
+
+ def test_rejects_javascript_protocol(self):
+ s = PlainSerializer(data={"name": "javascript:alert(1)", "description": ""})
+ assert not s.is_valid()
+ assert "name" in s.errors
+
+ def test_rejects_event_handler(self):
+ s = PlainSerializer(data={"name": "onclick=alert(1)", "description": ""})
+ assert not s.is_valid()
+ assert "name" in s.errors
+
+ def test_validation_raises_validation_error_type(self):
+ """End-to-end: raise_exception path surfaces a DRF ValidationError."""
+ s = PlainSerializer(data={"name": "<script>", "description": ""})
+ with pytest.raises(ValidationError):
+ s.is_valid(raise_exception=True)