Open-source personal finance manager. Self-hosted, privacy-first.
fix: reject unsafe regex rule patterns (#663)
seraphblade20105daf3b25ac8b
diff --git a/backend/app/services/rule_engine.py b/backend/app/services/rule_engine.pyindex 14674ce1..44b3657f 100644--- a/backend/app/services/rule_engine.py+++ b/backend/app/services/rule_engine.py@@ -16,6 +16,18 @@ def _strip_accents(text: str) -> str:return "".join(c for c in nfkd if not unicodedata.combining(c))+def compile_rule_regex(pattern: str) -> re.Pattern[str]:+ """Compile a regex using Securo's runtime normalization and safety policy."""+ effective_pattern = _strip_accents(pattern)+ try:+ compiled = re.compile(effective_pattern, re.IGNORECASE)+ except re.error as exc:+ raise ValueError("Invalid regular expression") from exc+ if compiled.search("") is not None:+ raise ValueError("Regular expression must not match an empty string")+ return compiled++def _normalize(text: str) -> str:"""Normalize text: uppercase and remove diacritics (accents)."""return _strip_accents(text.upper())@@ -80,9 +92,9 @@ def _match_condition(condition: dict, tx: "Transaction") -> bool:# normalized text, but keep its case: uppercasing a regex# inverts escape classes (\s -> \S, \b -> \B, \d -> \D) and# breaks inline flags. Case is already handled by IGNORECASE.- pattern = _strip_accents(str(value or ""))- return bool(re.search(pattern, tx_str, re.IGNORECASE))- except re.error:+ compiled = compile_rule_regex(str(value or ""))+ return compiled.search(tx_str) is not None+ except ValueError:return False# Numeric operatorsdiff --git a/backend/app/services/rule_service.py b/backend/app/services/rule_service.pyindex 52812323..30719250 100644--- a/backend/app/services/rule_service.py+++ b/backend/app/services/rule_service.py@@ -10,7 +10,11 @@ from app.models.category import Categoryfrom app.models.payee import Payeefrom app.models.transaction import Transactionfrom app.schemas.rule import RuleCreate, RuleExportPayload, RuleImportResponse, RuleUpdate-from app.services.rule_engine import evaluate_conditions, apply_rule_actions+from app.services.rule_engine import (+ apply_rule_actions,+ compile_rule_regex,+ evaluate_conditions,+)from app.services.category_service import DEFAULT_CATEGORIES_I18N@@ -63,6 +67,8 @@ async def _validate_rule_definition(op = _rule_item_value(condition, "op")if field not in _ALLOWED_CONDITION_FIELDS or op not in _ALLOWED_CONDITION_OPS:raise ValueError("Invalid rule condition")+ if op == "regex":+ compile_rule_regex(str(_rule_item_value(condition, "value") or ""))for action in actions or []:op = _rule_item_value(action, "op")diff --git a/backend/tests/test_new_rules_api.py b/backend/tests/test_new_rules_api.pyindex d8517395..19199c7c 100644--- a/backend/tests/test_new_rules_api.py+++ b/backend/tests/test_new_rules_api.py@@ -159,6 +159,98 @@ async def test_create_rule_no_match_reports_zero(assert response.json()["applied_count"] == 0+async def test_create_rule_rejects_unsafe_regex_before_persistence_or_history(+ client: AsyncClient, auth_headers, test_transactions, test_categories+):+ before_items = (+ await client.get("/api/transactions", headers=auth_headers)+ ).json()["items"]+ before_categories = {item["id"]: item["category_id"] for item in before_items}+ target_category = str(test_categories[1].id)+ rule_name = "Unsafe regex create"++ response = await client.post(+ "/api/rules",+ json={+ "name": rule_name,+ "conditions_op": "and",+ "conditions": [+ {"field": "description", "op": "regex", "value": "foo|"}+ ],+ "actions": [{"op": "set_category", "value": target_category}],+ "priority": 5,+ "is_active": True,+ "apply_to_existing": True,+ },+ headers=auth_headers,+ )++ persisted_rules = [+ rule+ for rule in (await client.get("/api/rules", headers=auth_headers)).json()+ if rule["name"] == rule_name+ ]+ after_items = (+ await client.get("/api/transactions", headers=auth_headers)+ ).json()["items"]+ changed_transactions = {+ item["description"]: {+ "before_category_id": before_categories[item["id"]],+ "after_category_id": item["category_id"],+ }+ for item in after_items+ if item["category_id"] != before_categories[item["id"]]+ }++ assert {+ "status": response.status_code,+ "detail": response.json().get("detail"),+ "applied_count": response.json().get("applied_count"),+ "persisted_conditions": [+ rule["conditions"] for rule in persisted_rules+ ],+ "changed_transactions": changed_transactions,+ } == {+ "status": 400,+ "detail": "Regular expression must not match an empty string",+ "applied_count": None,+ "persisted_conditions": [],+ "changed_transactions": {},+ }+++async def test_create_rule_rejects_malformed_regex(+ client: AsyncClient, auth_headers, test_categories+):+ rule_name = "Malformed regex create"+ response = await client.post(+ "/api/rules",+ json={+ "name": rule_name,+ "conditions": [+ {"field": "description", "op": "regex", "value": "["}+ ],+ "actions": [+ {"op": "set_category", "value": str(test_categories[0].id)}+ ],+ "is_active": True,+ },+ headers=auth_headers,+ )+ persisted_names = {+ rule["name"]+ for rule in (await client.get("/api/rules", headers=auth_headers)).json()+ }++ assert (+ response.status_code,+ response.json().get("detail"),+ rule_name in persisted_names,+ ) == (400, "Invalid regular expression", False)++@pytest.mark.asyncioasync def test_update_rule(client: AsyncClient, auth_headers, test_rules):rule_id = str(test_rules[0].id)@@ -249,6 +341,80 @@ async def test_update_rule_applies_to_existing_transactions(assert netflix["category_id"] == cat_food+async def test_update_rule_rejects_unsafe_regex_atomically(+ client: AsyncClient, auth_headers, test_transactions, test_categories+):+ create_response = await client.post(+ "/api/rules",+ json={+ "name": "Safe rule before unsafe update",+ "conditions_op": "and",+ "conditions": [+ {+ "field": "description",+ "op": "regex",+ "value": "ZZZ_NOMATCH",+ }+ ],+ "actions": [+ {"op": "set_category", "value": str(test_categories[1].id)}+ ],+ "priority": 5,+ "is_active": True,+ },+ headers=auth_headers,+ )+ assert create_response.status_code == 201+ rule_id = create_response.json()["id"]+ original_conditions = create_response.json()["conditions"]+ before_items = (+ await client.get("/api/transactions", headers=auth_headers)+ ).json()["items"]+ before_categories = {item["id"]: item["category_id"] for item in before_items}++ update_response = await client.patch(+ f"/api/rules/{rule_id}",+ json={+ "conditions": [+ {"field": "description", "op": "regex", "value": "foo|"}+ ]+ },+ headers=auth_headers,+ )++ persisted_rule = next(+ rule+ for rule in (await client.get("/api/rules", headers=auth_headers)).json()+ if rule["id"] == rule_id+ )+ after_items = (+ await client.get("/api/transactions", headers=auth_headers)+ ).json()["items"]+ changed_transactions = {+ item["description"]: {+ "before_category_id": before_categories[item["id"]],+ "after_category_id": item["category_id"],+ }+ for item in after_items+ if item["category_id"] != before_categories[item["id"]]+ }++ assert {+ "status": update_response.status_code,+ "detail": update_response.json().get("detail"),+ "applied_count": update_response.json().get("applied_count"),+ "persisted_conditions": persisted_rule["conditions"],+ "changed_transactions": changed_transactions,+ } == {+ "status": 400,+ "detail": "Regular expression must not match an empty string",+ "applied_count": None,+ "persisted_conditions": original_conditions,+ "changed_transactions": {},+ }++@pytest.mark.asyncioasync def test_update_rule_can_skip_existing_transactions(client: AsyncClient, auth_headers, test_transactions, test_categories,diff --git a/backend/tests/test_rule_engine.py b/backend/tests/test_rule_engine.pyindex 8c27bca6..46a08871 100644--- a/backend/tests/test_rule_engine.py+++ b/backend/tests/test_rule_engine.py@@ -4,6 +4,8 @@ from decimal import Decimalfrom datetime import date+import pytest+from app.models import Transactionfrom app.services.rule_engine import evaluate_conditions, apply_rule_actions@@ -128,6 +130,39 @@ def test_regex_accented_pattern_still_matches():assert evaluate_conditions("and", conditions, tx) is True+ "pattern",+ ["foo|", "|foo", "(foo|)", "(?:foo|)", "a*", ".*", "^", "$"],+)+def test_unsafe_persisted_regex_does_not_match_unrelated_transaction(pattern):+ conditions = [{"field": "description", "op": "regex", "value": pattern}]+ tx = make_tx(description="UNRELATED TRANSACTION")++ assert evaluate_conditions("and", conditions, tx) is False+++def test_empty_only_regex_is_inert_for_empty_transaction_field():+ conditions = [{"field": "description", "op": "regex", "value": "^$"}]+ tx = make_tx(description="")++ # Empty-matching regexes are disallowed for rule safety. Keeping legacy+ # definitions inert aligns runtime behavior with future save-time validation.+ assert evaluate_conditions("and", conditions, tx) is False+++def test_safe_alternation_regex_remains_selective():+ conditions = [{"field": "description", "op": "regex", "value": "foo|bar"}]++ assert evaluate_conditions("and", conditions, make_tx(description="foo")) is True+ assert evaluate_conditions("and", conditions, make_tx(description="bar")) is True+ assert (+ evaluate_conditions(+ "and", conditions, make_tx(description="UNRELATED TRANSACTION")+ )+ is False+ )++def test_amount_lt():conditions = [{"field": "amount", "op": "lt", "value": 50}]tx = make_tx(amount=Decimal("25.50"))diff --git a/backend/tests/test_rule_service.py b/backend/tests/test_rule_service.pyindex 853691e3..62379209 100644--- a/backend/tests/test_rule_service.py+++ b/backend/tests/test_rule_service.py@@ -9,7 +9,15 @@ from app.models.account import Accountfrom app.models.category import Categoryfrom app.models.payee import Payeefrom app.models.transaction import Transaction-from app.schemas.rule import RuleAction, RuleCondition, RuleCreate, RuleExportItem, RuleExportPayload, RuleUpdate+from app.schemas.rule import (+ RuleAction,+ RuleCondition,+ RuleConditionGroup,+ RuleCreate,+ RuleExportItem,+ RuleExportPayload,+ RuleUpdate,+)from app.schemas.transaction import TransactionUpdatefrom app.services.rule_service import (DuplicateRuleError,@@ -232,6 +240,124 @@ async def test_create_rule_rejects_category_outside_workspace()[email protected]("pattern", ["foo|", "a*", "^$"])+async def test_create_rule_rejects_empty_matching_regex(+ session: AsyncSession, test_user, test_workspace, pattern+):+ name = f"Unsafe regex {pattern}"+ error = None+ try:+ await create_rule(+ session,+ test_workspace.id,+ test_user.id,+ RuleCreate(+ name=name,+ conditions=[+ RuleCondition(field="description", op="regex", value=pattern)+ ],+ actions=[RuleAction(op="append_notes", value="#unsafe")],+ ),+ )+ except ValueError as exc:+ error = str(exc)++ persisted = {rule.name for rule in await get_rules(session, test_workspace.id)}+ assert (error, name in persisted) == (+ "Regular expression must not match an empty string",+ False,+ )+++async def test_create_rule_rejects_malformed_regex(+ session: AsyncSession, test_user, test_workspace+):+ name = "Malformed regex"+ error = None+ try:+ await create_rule(+ session,+ test_workspace.id,+ test_user.id,+ RuleCreate(+ name=name,+ conditions=[RuleCondition(field="description", op="regex", value="[")],+ actions=[RuleAction(op="append_notes", value="#malformed")],+ ),+ )+ except ValueError as exc:+ error = str(exc)++ persisted = {rule.name for rule in await get_rules(session, test_workspace.id)}+ assert (error, name in persisted) == ("Invalid regular expression", False)+++ "pattern",+ ["foo|bar", "^NETFLIX", "PIX.*RECEBIDO", r"\bFOO\b", "foo.*", ".+", "^.+$"],+)+async def test_create_rule_accepts_safe_regex(+ session: AsyncSession, test_user, test_workspace, pattern+):+ await create_rule(+ session,+ test_workspace.id,+ test_user.id,+ RuleCreate(+ name=f"Safe regex {pattern}",+ conditions=[RuleCondition(field="description", op="regex", value=pattern)],+ actions=[RuleAction(op="append_notes", value="#safe")],+ ),+ )++++async def test_create_rule_rejects_empty_matching_regex_in_condition_group(+ session: AsyncSession, test_user, test_workspace+):+ name = "Unsafe grouped regex"+ error = None+ try:+ await create_rule(+ session,+ test_workspace.id,+ test_user.id,+ RuleCreate(+ name=name,+ conditions_op="or",+ conditions=[+ RuleCondition(+ field="description", op="contains", value="SAFE"+ ),+ RuleConditionGroup(+ op="or",+ conditions=[+ RuleCondition(+ field="description", op="regex", value="foo|"+ ),+ RuleCondition(+ field="description", op="contains", value="OTHER"+ ),+ ],+ ),+ ],+ actions=[RuleAction(op="append_notes", value="#grouped")],+ ),+ )+ except ValueError as exc:+ error = str(exc)++ persisted = {rule.name for rule in await get_rules(session, test_workspace.id)}+ assert (error, name in persisted) == (+ "Regular expression must not match an empty string",+ False,+ )++@pytest.mark.asyncioasync def test_update_rule_rejects_payee_outside_workspace(session: AsyncSession, test_user, test_workspace@@ -302,6 +428,53 @@ async def test_import_rules_skips_invalid_rules(assert result.skipped == 2++async def test_import_rules_skips_unsafe_and_malformed_regexes(+ session: AsyncSession, test_user, test_workspace+):+ payload = RuleExportPayload(+ rules=[+ RuleExportItem(+ name="Unsafe regex import",+ conditions=[+ RuleCondition(field="description", op="regex", value="foo|")+ ],+ actions=[RuleAction(op="append_notes", value="#unsafe")],+ ),+ RuleExportItem(+ name="Malformed regex import",+ conditions=[RuleCondition(field="description", op="regex", value="[")],+ actions=[RuleAction(op="append_notes", value="#malformed")],+ ),+ RuleExportItem(+ name="Safe regex import",+ conditions=[+ RuleCondition(field="description", op="regex", value="foo|bar")+ ],+ actions=[RuleAction(op="append_notes", value="#safe")],+ ),+ ]+ )++ result = await import_rules(+ session, test_workspace.id, test_user.id, payload, overwrite=True+ )+ persisted = {+ rule.name: rule.conditions[0]["value"]+ for rule in await get_rules(session, test_workspace.id)+ }++ assert (+ result.imported,+ result.skipped,+ persisted,+ ) == (+ 1,+ 2,+ {"Safe regex import": "foo|bar"},+ )+@pytest.mark.asyncioasync def test_set_description_validation_and_export_compatibility(session: AsyncSession, test_user, test_workspace