DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(search): don't let flagged long references bypass Moderation
Rudy Celeklid57770ed658d
diff --git a/deeptutor/services/search/source_filter.py b/deeptutor/services/search/source_filter.pyindex 9ad5b4c74..9585b352c 100644--- a/deeptutor/services/search/source_filter.py+++ b/deeptutor/services/search/source_filter.py@@ -224,18 +224,23 @@ def _moderation_rejections(Fail open on transport/API errors so a Moderation outage does not blankevery web-search turn — heuristics already ran."""- payloads = list(dict.fromkeys(text.strip()[:4000] for text in texts if text.strip()))- if not payloads or not api_key:+ # Key by each exact submitted text so a caller can look up its own row text.+ # Re-stripping here (text.strip()[:4000]) silently desynced the key from the+ # submitted text when the caller's text ended in whitespace — e.g. a long+ # reference whose own [:4000] cut landed on a space — so the lookup missed+ # and a flagged reference failed open. Cap only the payload sent to the API.+ unique_texts = list(dict.fromkeys(text for text in texts if text.strip()))+ if not unique_texts or not api_key:return {}requester = request_moderation or _request_openai_moderationtry:- flagged = requester(payloads, api_key=api_key)- if not isinstance(flagged, (list, tuple)) or len(flagged) != len(payloads):+ flagged = requester([text[:4000] for text in unique_texts], api_key=api_key)+ if not isinstance(flagged, (list, tuple)) or len(flagged) != len(unique_texts):raise ValueError("Moderation returned an unexpected result count")except Exception as exc: # noqa: BLE001 — network / JSON / provider quirks_logger.warning("OpenAI Moderation skipped after error: %s", exc)return {}- return {text: bool(decision) for text, decision in zip(payloads, flagged, strict=True)}+ return {text: bool(decision) for text, decision in zip(unique_texts, flagged, strict=True)}def _request_openai_moderation(texts: list[str], *, api_key: str) -> list[bool]:diff --git a/tests/services/search/test_web_search_runtime.py b/tests/services/search/test_web_search_runtime.pyindex 1bed0621d..2012d25c7 100644--- a/tests/services/search/test_web_search_runtime.py+++ b/tests/services/search/test_web_search_runtime.py@@ -688,6 +688,49 @@ def test_source_filter_moderation_fails_open_on_errors() -> None:assert filtered.metadata["source_filter"]["removed_citations"] == 0+def test_source_filter_moderation_flags_long_reference_truncated_on_whitespace() -> None:+ # A reference whose text exceeds the 4000-char moderation cap and whose cut+ # lands on whitespace must still be dropped when Moderation flags it. The+ # decision dict was keyed by text.strip()[:4000] while the lookup used the+ # unstripped row text, so the flag silently missed (fail-open).+ flagged_texts: list[str] = []++ def _flag_all(texts: list[str], *, api_key: str) -> list[bool]:+ flagged_texts.extend(texts)+ return [True for _ in texts]++ long_content = "a " * 3000 # 6000 chars; the [:4000] cut lands on a space++ response = WebSearchResponse(+ query="news",+ answer="",+ provider="test",+ citations=[+ Citation(+ id=1,+ reference="[1]",+ url="https://news.example/a",+ title="flag",+ snippet="this",+ content=long_content,+ ),+ ],+ search_results=[],+ )++ filtered = filter_web_search_response(+ response,+ content_filtering=False,+ use_moderation=True,+ moderation_api_key="sk-test",+ request_moderation=_flag_all,+ )++ assert flagged_texts, "moderation requester was never called"+ assert [c.id for c in filtered.citations] == []+ assert "moderation_flagged" in filtered.metadata["source_filter"]["rejected_reasons"]++def test_settings_from_config_defaults_and_educational_flag(monkeypatch) -> None:monkeypatch.delenv("OPENAI_API_KEY", raising=False)monkeypatch.delenv("DEEPTUTOR_OPENAI_API_KEY", raising=False)