LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
import pytest
from rest_framework.serializers import ValidationError
from utils.input_sanitizer import (
validate_name_field,
validate_no_html_tags,
validate_safe_text,
)
class TestValidateNoHtmlTags:
def test_clean_input_passes(self):
assert validate_no_html_tags("Hello World") == "Hello World"
def test_allows_normal_special_chars(self):
assert (
validate_no_html_tags("My workflow (v2), test - final")
== "My workflow (v2), test - final"
)
def test_allows_numbers_and_punctuation(self):
assert validate_no_html_tags("Test 123 & more!") == "Test 123 & more!"
def test_rejects_script_tag(self):
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
validate_no_html_tags("<script>alert(1)</script>")
def test_rejects_img_tag_via_event_handler(self):
with pytest.raises(
ValidationError, match="must not contain event handler attributes"
):
validate_no_html_tags("<img src=x onerror=alert(1)>")
def test_rejects_iframe_tag(self):
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
validate_no_html_tags("<iframe src=//evil.example></iframe>")
def test_rejects_svg_tag(self):
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
validate_no_html_tags("<svg/onload=alert(1)>")
def test_rejects_closing_script_tag(self):
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
validate_no_html_tags("</script>")
def test_rejects_script_tag_with_inner_space(self):
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
validate_no_html_tags("< script >alert(1)")
def test_rejects_unclosed_script_tag(self):
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
validate_no_html_tags("<script")
def test_allows_inert_markup(self):
assert validate_no_html_tags("<div>content</div>") == "<div>content</div>"
assert validate_no_html_tags("Use <br> to break lines.") == (
"Use <br> to break lines."
)
def test_allows_less_than_with_number(self):
assert validate_no_html_tags("a < 3") == "a < 3"
def test_allows_comparison_without_space(self):
assert validate_no_html_tags("rows where qty <threshold") == (
"rows where qty <threshold"
)
assert validate_no_html_tags("if x<y then return 1") == "if x<y then return 1"
def test_allows_angle_bracket_placeholders(self):
assert validate_no_html_tags("extract the <invoice_no> field") == (
"extract the <invoice_no> field"
)
def test_allows_currency_range(self):
assert validate_no_html_tags("total is < 500 USD, tax > 5%") == (
"total is < 500 USD, tax > 5%"
)
def test_rejects_javascript_protocol(self):
with pytest.raises(ValidationError, match="must not contain dangerous URI protocols"):
validate_no_html_tags("javascript:alert(1)")
def test_rejects_javascript_protocol_with_spaces(self):
with pytest.raises(ValidationError, match="must not contain dangerous URI protocols"):
validate_no_html_tags("javascript :alert(1)")
def test_rejects_javascript_protocol_case_insensitive(self):
with pytest.raises(ValidationError, match="must not contain dangerous URI protocols"):
validate_no_html_tags("JAVASCRIPT:alert(1)")
def test_rejects_data_uri(self):
with pytest.raises(ValidationError, match="must not contain dangerous URI protocols"):
validate_no_html_tags("data:text/html,alert(1)")
def test_rejects_data_uri_with_mime_type(self):
with pytest.raises(ValidationError, match="must not contain dangerous URI protocols"):
validate_no_html_tags("data:application/javascript,code")
def test_allows_inert_data_uri(self):
value = "See data:image/png;base64,iVBORw0KGgo="
assert validate_no_html_tags(value) == value
def test_allows_benign_data_colon_in_text(self):
assert validate_no_html_tags("Input data: JSON format") == "Input data: JSON format"
def test_allows_data_colon_without_mime(self):
assert validate_no_html_tags("my data: 100 records") == "my data: 100 records"
def test_rejects_vbscript_protocol(self):
with pytest.raises(ValidationError, match="must not contain dangerous URI protocols"):
validate_no_html_tags("vbscript:MsgBox")
def test_rejects_event_handler(self):
with pytest.raises(
ValidationError, match="must not contain event handler attributes"
):
validate_no_html_tags("onclick=alert(1)")
def test_rejects_event_handler_with_spaces(self):
with pytest.raises(
ValidationError, match="must not contain event handler attributes"
):
validate_no_html_tags("onerror =alert(1)")
def test_rejects_event_handler_case_insensitive(self):
with pytest.raises(
ValidationError, match="must not contain event handler attributes"
):
validate_no_html_tags("ONLOAD=alert(1)")
def test_allows_benign_connection_text(self):
assert validate_no_html_tags("connection=ok") == "connection=ok"
def test_allows_onboarding_text(self):
assert validate_no_html_tags("onboarding = enabled") == "onboarding = enabled"
def test_allows_oncall_text(self):
assert validate_no_html_tags("oncall = primary") == "oncall = primary"
def test_allows_condition_text(self):
assert validate_no_html_tags("condition = good") == "condition = good"
def test_allows_spaced_event_word(self):
assert validate_no_html_tags("on error = retry") == "on error = retry"
def test_allows_spaced_event_word_without_padding(self):
assert validate_no_html_tags("on change=true") == "on change=true"
def test_rejects_event_handler_after_semicolon(self):
with pytest.raises(
ValidationError, match="must not contain event handler attributes"
):
validate_no_html_tags("foo;onclick=alert(1)")
def test_rejects_event_handler_after_ampersand(self):
with pytest.raises(
ValidationError, match="must not contain event handler attributes"
):
validate_no_html_tags("foo&onerror=alert(1)")
def test_custom_field_name_in_error(self):
with pytest.raises(ValidationError, match="Workflow name"):
validate_no_html_tags("<script>", field_name="Workflow name")
class TestValidateNameField:
def test_clean_name_passes(self):
assert validate_name_field("My Workflow") == "My Workflow"
def test_strips_whitespace(self):
assert validate_name_field(" hello ") == "hello"
def test_rejects_empty_after_strip(self):
with pytest.raises(ValidationError, match="must not be empty"):
validate_name_field(" ")
def test_rejects_html_tags(self):
with pytest.raises(ValidationError, match="must not contain HTML or script tags"):
validate_name_field("<script>alert(1)</script>")
def test_allows_hyphens_and_underscores(self):
assert validate_name_field("my-workflow_v2") == "my-workflow_v2"
def test_allows_periods(self):
assert validate_name_field("config.v2") == "config.v2"
def test_allows_parentheses_and_commas(self):
assert validate_name_field("Test (v2), final") == "Test (v2), final"
def test_custom_field_name_in_error(self):
with pytest.raises(ValidationError, match="Tool name"):
validate_name_field(" ", field_name="Tool name")
class TestValidateSafeText:
"""The strict allow-list used for key names/descriptions.
Unlike its block-listing siblings above, anything outside
``SAFE_TEXT_PATTERN`` is rejected — so these cases pin what the allow-list
actually permits, including the apostrophe it deliberately keeps.
"""
def test_clean_text_passes(self):
assert validate_safe_text("Prod deployment key") == "Prod deployment key"
def test_strips_whitespace(self):
assert validate_safe_text(" x ") == "x"
def test_rejects_whitespace_only(self):
with pytest.raises(ValidationError, match="cannot be empty"):
validate_safe_text(" ")
def test_rejects_html_tags(self):
with pytest.raises(ValidationError, match="Only alphanumeric characters"):
validate_safe_text("<script>alert(1)</script>")
def test_rejects_ampersand(self):
with pytest.raises(ValidationError, match="Only alphanumeric characters"):
validate_safe_text("Billing & Ops")
def test_rejects_double_quote(self):
with pytest.raises(ValidationError, match="Only alphanumeric characters"):
validate_safe_text('Say "hi"')
def test_rejects_backtick(self):
with pytest.raises(ValidationError, match="Only alphanumeric characters"):
validate_safe_text("`whoami`")
def test_allows_apostrophe(self):
# Documented carve-out: real names contain it, so single-quoted HTML
# attribute contexts must escape at render time (see module comment).
assert validate_safe_text("Client's Prod Key") == "Client's Prod Key"
def test_allows_punctuation_subset(self):
value = "Key-1_v2 (prod), region: us/west."
assert validate_safe_text(value) == value