Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(python-sdk): merge overlapping PII spans before redacting (#1444)

Paco Cartones46adff26be80
diff --git a/sdk/python/src/openlit/guard/pii.py b/sdk/python/src/openlit/guard/pii.py
index 5708c8cc..0b90ffd5 100644
--- a/sdk/python/src/openlit/guard/pii.py
+++ b/sdk/python/src/openlit/guard/pii.py
@@ -107,13 +107,27 @@ class PII(Guard):
transformed: Optional[str] = None
if self._action == GuardAction.REDACT:
- sorted_matches = sorted(matches, key=lambda x: x[1].start(), reverse=True)
+ # Patterns can match overlapping spans of the same text. Every
+ # span's offsets refer to the original string, so replacing them
+ # one at a time lets an earlier replacement resize the string out
+ # from under a later one -- the later slice then stops short and
+ # leaves the tail of the value in the output. Merge overlapping
+ # spans first so each replacement covers a region no other touches.
+ ordered = sorted(
+ enumerate(matches),
+ key=lambda x: (x[1][1].start(), -x[1][1].end(), -x[0]),
+ )
+ spans: List[Tuple[str, int, int]] = []
+ for _, (label, m) in ordered:
+ if spans and m.start() < spans[-1][2]:
+ prev_label, prev_start, prev_end = spans[-1]
+ spans[-1] = (prev_label, prev_start, max(prev_end, m.end()))
+ else:
+ spans.append((label, m.start(), m.end()))
result_text = text
- for label, m in sorted_matches:
+ for label, start, end in reversed(spans):
result_text = (
- result_text[: m.start()]
- + f"[REDACTED:{label}]"
- + result_text[m.end() :]
+ result_text[:start] + f"[REDACTED:{label}]" + result_text[end:]
)
transformed = result_text
diff --git a/sdk/python/tests/test_guard_pii.py b/sdk/python/tests/test_guard_pii.py
index 04f7a3e0..d5b02826 100644
--- a/sdk/python/tests/test_guard_pii.py
+++ b/sdk/python/tests/test_guard_pii.py
@@ -119,6 +119,37 @@ class TestPIIRedaction:
assert "[email protected]" not in result.transformed_text
assert "111-22-3333" not in result.transformed_text
+ def test_overlapping_spans_redacted_cleanly(self):
+ """Overlapping matches (env-secret covering an API key) redact as one span."""
+ guard = PII(action="redact")
+ result = guard.evaluate("config: api_key=sk-proj-abcdefghij1234567890 done")
+ assert result.transformed_text == "config: [REDACTED:env-secret] done"
+
+ def test_overlapping_spans_leave_no_secret_tail(self):
+ """An inner match must not leave the tail of the value in the output."""
+ guard = PII(action="redact")
+ result = guard.evaluate("token=1.1.1.1secretos")
+ assert result.transformed_text == "[REDACTED:env-secret]"
+ assert "secretos" not in result.transformed_text
+
+ def test_disjoint_spans_still_redact_separately(self):
+ """Non-overlapping matches keep their own placeholders."""
+ guard = PII(action="redact")
+ result = guard.evaluate("contact [email protected] or call 415-555-0100")
+ assert (
+ result.transformed_text
+ == "contact [REDACTED:email] or call [REDACTED:phone-us]"
+ )
+
+ def test_adjacent_spans_still_redact_separately(self):
+ """Touching but non-overlapping matches are not merged into one span."""
+ guard = PII(action="redact")
+ result = guard.evaluate("SK" + "a" * 32 + "github_pat_" + "b" * 22)
+ assert (
+ result.transformed_text
+ == "[REDACTED:twilio-api-key][REDACTED:github-fine-grained]"
+ )
+
class TestPIIWarn: # pylint: disable=too-few-public-methods
"""``warn`` action emits an event without transforming the text."""