Enterprise agent platform for building, deploying, and operating AI teammates.
feat(tools): redact personal data from web search queries (toby) (#3800)
OliverBryant403681d576e7
diff --git a/src/xagent/core/agent/trace.py b/src/xagent/core/agent/trace.pyindex 578f753ca..9071007c5 100644--- a/src/xagent/core/agent/trace.py+++ b/src/xagent/core/agent/trace.py@@ -3,6 +3,7 @@import inspectimport jsonimport logging+import refrom abc import ABC, abstractmethodfrom datetime import datetime, timezonefrom enum import Enum@@ -131,6 +132,12 @@ _TRUNCATABLE_CONTENT_FIELDS = frozenset()+# Opens every truncation marker the reducers below write+# (``...[truncated N chars]``, ``...[truncated N middle items]``). The console+# sink keys on it to recognise text that was cut upstream.+_TRUNCATION_MARKER = "...[truncated "++def truncate_for_trace(value: Any, max_bytes: Optional[int] = None) -> Any:"""Recursive per-leaf truncation of a trace value.@@ -196,7 +203,7 @@ def _truncate_string(value: str, budget: int) -> str:slack = min(30, max(8, budget // 4))head_budget = max(1, budget - slack)head = encoded[:head_budget].decode("utf-8", errors="ignore")- return f"{head}...[truncated {len(value) - len(head)} chars]"+ return f"{head}{_TRUNCATION_MARKER}{len(value) - len(head)} chars]"def _placeholder(message: str) -> Dict[str, str]:@@ -229,7 +236,7 @@ def _reduce_text_list(value: Any, budget: int) -> Any:[_reduce_text(v, per_item) for v in value[-keep_tail:]] if keep_tail else [])omitted = len(value) - keep_head - keep_tail- marker = f"...[truncated {omitted} middle items]"+ marker = f"{_TRUNCATION_MARKER}{omitted} middle items]"candidate: List[Any] = head + [marker] + tailif _serialized_size(candidate) <= budget:return candidate@@ -627,7 +634,7 @@ def _trim_subtree(value: Any, max_bytes: int, _depth: int = 0) -> Any:# U+FFFD replacement chars — otherwise len(head) inflates and the# reported truncation count becomes inaccurate (can even go negative).head = encoded[:max_bytes].decode("utf-8", errors="ignore")- return f"{head}...[truncated {len(value) - len(head)} chars]"+ return f"{head}{_TRUNCATION_MARKER}{len(value) - len(head)} chars]"if isinstance(value, list):per_item = max(1, max_bytes // max(1, len(value)))@@ -1015,7 +1022,7 @@ def _shrink_string_leaf(marked = ""for _ in range(_LOG_TRUNCATE_FIT_PASSES):head = encoded[:cut].decode("utf-8", errors="ignore")- marked = f"{head}...[truncated {len(value) - len(head)} chars]"+ marked = f"{head}{_TRUNCATION_MARKER}{len(value) - len(head)} chars]"marked_width = _leaf_width(marked, depth)if marked_width <= spendable or cut == 0:break@@ -1057,7 +1064,9 @@ def _shrink_node(return too_deepif isinstance(value, str):- return _shrink_string_leaf(value, remaining, depth, slack)+ return _shrink_string_leaf(+ _withhold_truncated_query_text(value), remaining, depth, slack+ )if isinstance(value, dict):node_id = id(value)@@ -1067,7 +1076,10 @@ def _shrink_node(seen.add(node_id)shrunk: Dict[Any, Any] = {}remaining[0] -= 2 # rendered braces- for key, item in value.items():+ # Redact a search call's query *before* its leaves are measured or cut,+ # so the byte budget can never leave a fragment of the original query.+ projected = _project_search_call(value)+ for key, item in projected.items():key_cost = _rendered_width(f"{key!r}: , ")# A key is charged in full and can't be truncated, so decide# before spending: stopping here instead of after keeps the copy@@ -1112,6 +1124,178 @@ def _shrink_node(return value+# Tool names whose ``query`` argument is sent to an external search provider.+# The single definition: a new search tool (e.g. Brave) is added here, and+# ``tests/core/tools/test_web_search_redaction_providers.py`` fails if a+# registered search tool name is missing from the set.+SEARCH_TOOL_NAMES = frozenset(+ {"web_search", "tavily_web_search", "zhipu_web_search", "exa_web_search"}+)++# Keys under which a tool call carries its arguments, across the shapes the+# trace sees: ``tool_params`` (tool events), ``args`` (normalised calls and the+# checkpoint ledger), ``arguments`` (OpenAI ``function``), ``input`` / ``params``+# / ``parameters`` (other adapters).+_SEARCH_ARG_KEYS = ("tool_params", "args", "arguments", "input", "params", "parameters")+++def _search_call_name(node: Dict[Any, Any]) -> Optional[str]:+ for key in ("tool_name", "name"):+ name = node.get(key)+ if isinstance(name, str):+ return name+ return None+++# Fixed replacements. Neither contains any of the text it stands for.+_SEARCH_ARGS_WITHHELD = "[search arguments withheld]"+_QUERY_TEXT_WITHHELD = "[truncated text containing a query withheld]"++# The bare word ``query`` followed, after any run of quotes and backslashes (so+# at any JSON/repr escaping depth: ``"query":``, ``'query':``, ``\\"query\\":``),+# by a colon: how a ``query`` key looks inside serialised arguments.+_QUERY_KEY_RE = re.compile(r"query[\\\"']*\s*:")+# A search tool name right after a quote or backslash (how a name looks inside+# serialised JSON or a repr).+_SEARCH_NAME_RE = re.compile(+ r"[\\\"'](?:" + "|".join(sorted(SEARCH_TOOL_NAMES)) + r")\b"+)+++def _withhold_truncated_query_text(text: str) -> str:+ """Make a string leaf that holds serialised search arguments safe to log.++ ``_shrink_message`` dumps a dict or list message value (a legacy+ ``function_call``, Anthropic content blocks) to a string, and may then cut+ it, so the structural projection never sees the call. A string that holds a+ ``query:`` key, or names a search tool in quotes (so also the+ ``{"input": <raw text>}`` fallback), is handled in two ways:++ * cut (carries a truncation marker): replaced wholesale, because a cut+ value leaves a fragment no recogniser can match (``jane.doe@exa``). A+ query value can only follow its key, so a cut before the key carries+ none of it;+ * complete: when it names a search tool, every entity in it is redacted.++ The wholesale rule over-masks truncated text that merely mentions+ ``query:`` or quotes a search tool name (prose, other tools' ``query``+ arguments); it affects only the console line.+ """+ has_key = _QUERY_KEY_RE.search(text) is not None+ names_tool = _SEARCH_NAME_RE.search(text) is not None+ if not (has_key or names_tool):+ return text+ if _TRUNCATION_MARKER in text:+ return _QUERY_TEXT_WITHHELD+ if names_tool:+ from ..tools.core.search_query_redaction import redact_search_query++ return redact_search_query(text).text+ return text+++def _redact_search_value(value: Any, key: Any = None) -> Any:+ """Redact every string leaf of a search call's arguments (copy on change)."""+ from ..tools.core.search_query_redaction import redact_search_query++ if isinstance(value, str):+ if _TRUNCATION_MARKER in value:+ return _SEARCH_ARGS_WITHHELD+ text = redact_search_query(value).text+ return value if text == value else text+ if isinstance(value, dict):+ out = {k: _redact_search_value(v, k) for k, v in value.items()}+ return out if any(out[k] is not value[k] for k in value) else value+ if isinstance(value, list):+ items = [_redact_search_value(v) for v in value]+ return items if any(a is not b for a, b in zip(items, value)) else value+ if key == "query" and value is not None and not isinstance(value, bool):+ return redact_search_query(str(value)).text+ return value+++def _redact_search_args(args: Any) -> Any:+ """Return a search call's arguments with every string leaf redacted.++ Console projection only; the provider-bound contract stays "the ``query``+ argument". Every string is redacted, not just ``query``, because arguments+ that failed to parse reach the tool as ``{"input": <raw text>}``. A string+ that is not a JSON object/array (malformed, or cut upstream) is replaced+ wholesale by a fixed marker.+ """+ if isinstance(args, str):+ if not args.strip():+ return args+ try:+ parsed = json.loads(args)+ except ValueError:+ return _SEARCH_ARGS_WITHHELD+ if isinstance(parsed, (dict, list)):+ return json.dumps(_redact_search_value(parsed), ensure_ascii=False)+ return _SEARCH_ARGS_WITHHELD+ return _redact_search_value(args)+++def _project_search_call(node: Dict[Any, Any]) -> Dict[Any, Any]:+ """Return ``node`` with a search call's query redacted, one level only.++ ``node`` is a search call when its ``tool_name`` / ``name`` is a search+ tool; an OpenAI ``{"function": {"name": ..., "arguments": ...}}`` wrapper+ is handled when the caller reaches its ``function`` child. The caller walks+ the children itself, so this never recurses. Returns ``node`` itself when+ nothing needs to change.+ """+ result = node+ name = _search_call_name(node)+ if name in SEARCH_TOOL_NAMES:+ for key in _SEARCH_ARG_KEYS:+ if key in node:+ redacted = _redact_search_args(node[key])+ if redacted is not node[key]:+ if result is node:+ result = dict(node)+ result[key] = redacted+ return result+++def _project_search_calls(+ value: Any, depth: int = 0, seen: Optional[Set[int]] = None+) -> Any:+ """Copy-on-write walk redacting every search call in ``value``.++ Used only where the byte-budgeted shrink walk does not run+ (``max_bytes <= 0``); depth is capped and cycles are cut.+ """+ if seen is None:+ seen = set()+ if depth >= _MAX_TRACE_DEPTH:+ return value+ if isinstance(value, str):+ return _withhold_truncated_query_text(value)+ if isinstance(value, dict):+ if id(value) in seen:+ return value+ seen.add(id(value))+ projected = _project_search_call(value)+ out = projected+ for key, item in projected.items():+ new = _project_search_calls(item, depth + 1, seen)+ if new is not item:+ if out is value:+ out = dict(projected)+ out[key] = new+ seen.discard(id(value))+ return out+ if isinstance(value, list):+ if id(value) in seen:+ return value+ seen.add(id(value))+ items = [_project_search_calls(item, depth + 1, seen) for item in value]+ seen.discard(id(value))+ return items if any(a is not b for a, b in zip(items, value)) else value+ return value++def _render_event_data_for_log(data: Any, max_bytes: Optional[int] = None) -> str:"""Render ``event.data`` into a size-bounded string for console logging.@@ -1143,7 +1327,7 @@ def _render_event_data_for_log(data: Any, max_bytes: Optional[int] = None) -> stmax_bytes = get_max_trace_payload_bytes()if max_bytes <= 0:- return f"{data}"+ return f"{_project_search_calls(data)}"rendered = f"{_shrink_within_budget(data, max_bytes)}"encoded = rendered.encode("utf-8", errors="replace")@@ -1152,7 +1336,7 @@ def _render_event_data_for_log(data: Any, max_bytes: Optional[int] = None) -> st# Slice on a byte boundary and drop an incomplete trailing multi-byte# char, matching the string-leaf truncation above.head = encoded[:max_bytes].decode("utf-8", errors="ignore")- return f"{head}...[truncated {len(rendered) - len(head)} chars]"+ return f"{head}{_TRUNCATION_MARKER}{len(rendered) - len(head)} chars]"class ConsoleTraceHandler(BaseTraceHandler):diff --git a/src/xagent/core/tools/core/exa_web_search.py b/src/xagent/core/tools/core/exa_web_search.pyindex 19b62abfe..467d617d2 100644--- a/src/xagent/core/tools/core/exa_web_search.py+++ b/src/xagent/core/tools/core/exa_web_search.py@@ -8,6 +8,8 @@ import loggingimport osfrom typing import Any, Dict, List, Optional+from .web_search_guard import run_guarded_search+logger = logging.getLogger(__name__)@@ -50,6 +52,40 @@ class ExaWebSearchCore:Returns:List of search results with title, link, snippet and optionally content"""+ return await run_guarded_search(+ "exa",+ query,+ lambda redacted: self._search_redacted(+ redacted,+ num_results=num_results,+ search_type=search_type,+ content_mode=content_mode,+ category=category,+ include_domains=include_domains,+ exclude_domains=exclude_domains,+ include_text=include_text,+ exclude_text=exclude_text,+ start_published_date=start_published_date,+ end_published_date=end_published_date,+ ),+ len,+ )++ async def _search_redacted(+ self,+ query: str,+ num_results: int,+ search_type: str,+ content_mode: str,+ category: Optional[str],+ include_domains: Optional[List[str]],+ exclude_domains: Optional[List[str]],+ include_text: Optional[List[str]],+ exclude_text: Optional[List[str]],+ start_published_date: Optional[str],+ end_published_date: Optional[str],+ ) -> List[Dict[str, str]]:+ """Run the search; ``query`` has already been redacted."""logger.info("🔍 Exa web search query='%s' num_results=%s type=%s content=%s",query,diff --git a/src/xagent/core/tools/core/search_query_redaction.py b/src/xagent/core/tools/core/search_query_redaction.pynew file mode 100644index 000000000..712f9df3d--- /dev/null+++ b/src/xagent/core/tools/core/search_query_redaction.py@@ -0,0 +1,366 @@+"""Redact personal data from web search queries.++The model writes search queries from conversation content, so a query can carry+an email address, phone number, street address or government identifier. This+module replaces those with typed placeholders before the query leaves the+process, so the query still reads as a search (``[EMAIL]``, ``[PHONE]``,+``[ADDRESS]``, ``[TFN]``, ``[MEDICARE]``, ``[ABN]``).++Design constraints:++* Pure and cheap: regular expressions only, no model, no network, no new+ dependency. Every pattern is bounded (no nested open-ended quantifiers) so a+ hostile query cannot trigger catastrophic backtracking.+* Names of people and organisations are deliberately kept: searching for+ public figures and businesses is normal use.+* TFN, Medicare and ABN candidates are replaced only when their checksum+ passes, so years, prices, order numbers and product codes survive.+* :func:`redact_search_query` never raises. If redaction itself fails, it fails+ closed: the whole query is replaced by a fixed placeholder instead of being+ passed through unredacted.++Recognised and not recognised+-----------------------------+Email: the local part and each domain label may contain any Unicode letter or+digit (RFC 6531), the domain may use an ASCII or ideographic dot, and a quoted+local part is accepted. A local part or label is either a run of non-CJK+characters or a run of CJK characters (Han, kana, hangul), never a mix, so an+address inside CJK text does not absorb the words beside it; a local part or+label made only of CJK characters still absorbs adjacent CJK text. The last+domain label must be letters only (at least two) or ``xn--`` punycode, so a+version specifier such as ``[email protected]`` is not an address, but one whose+last dotted part is alphabetic (``[email protected]``) is replaced. The+domain must contain at least one dot, so ``user@localhost`` and ``user@[IP]``+are not recognised, nor is an address with spaces around the ``@``. Not+replaced: ``git@host`` (the ssh user of git hosts) and an address followed by+``:`` and a path (scp-style remotes such as ``deploy@host:/srv``).++Phone: an international number starting with ``+`` (or the full-width ``+``),+checked against the structure of its country code (+61: 9 national digits,++1: 10, +44: 10; any other code: at most 15 digits in total), with an optional+trunk prefix (``(0)`` or a leading ``0``). Exactly the digits that belong to+the number are replaced; digits that follow it stay in the query. Also+Australian mobile (04xx xxx xxx, 04 xxxx xxxx, separators space, dot or+hyphen) and landline ((0x) xxxx xxxx), and US/Canada style ten-digit numbers+with separators. An extension (``ext 12``, ``x12``) is kept. 13xx/1300/1800+business numbers are not recognised.++TFN, ABN, Medicare: replaced only when the number stands alone (not glued to+letters, digits or a hyphen, and not part of a decimal or thousands figure),+is written with single spaces, hyphens or nothing between its groups, and its+checksum passes. Numbers glued to letters, written with dots, or grouped+irregularly are not recognised, so order numbers, versions and prices survive.++Address: an English-style street address (house number with optional unit or+level, a capitalised street name and a street-type word), or any-case street+name followed by ``<suburb> <STATE> <postcode>``; and PO Box, GPO Box, Locked+Bag and Private Bag numbers. A suburb or postcode on its own is not replaced.+Lowercase street names without a suburb and state, addresses in other+languages, and addresses that do not start with a house number are not+recognised.++Not covered: names of people and organisations; text the model restates in its+reply; search tool arguments other than ``query`` (for example Exa's+``include_text``); results returned by a provider. In console trace lines, a+search call's arguments that were cut or cannot be parsed are replaced whole by+a fixed marker, never partly shown.+"""++from __future__ import annotations++import logging+import re+from typing import Callable, List, Match, NamedTuple, Tuple++logger = logging.getLogger(__name__)++EMAIL = "email"+ADDRESS = "address"+ABN = "abn"+TFN = "tfn"+MEDICARE = "medicare"+PHONE = "phone"+REDACTION_ERROR = "redaction_error"++_PLACEHOLDERS = {+ EMAIL: "[EMAIL]",+ ADDRESS: "[ADDRESS]",+ ABN: "[ABN]",+ TFN: "[TFN]",+ MEDICARE: "[MEDICARE]",+ PHONE: "[PHONE]",+}+_FAILED_PLACEHOLDER = "[REDACTED]"+++class RedactionResult(NamedTuple):+ """Outcome of :func:`redact_search_query`."""++ text: str+ # Entity types that were replaced, de-duplicated, in the order the+ # recognisers ran. Values are the module constants (``"email"``, ...).+ entities: Tuple[str, ...]+++# --- checksums -------------------------------------------------------------+++def _digits(value: str) -> List[int]:+ return [int(c) for c in value if c.isdigit()]+++def valid_abn(value: str) -> bool:+ """ABN check: subtract 1 from the first digit, weight, sum, mod 89 == 0.++ Algorithm as published by the Australian Business Register:+ https://abr.business.gov.au/Help/AbnFormat+ (weights 10, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19).+ """+ d = _digits(value)+ if len(d) != 11:+ return False+ d[0] -= 1+ weights = (10, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19)+ return sum(w * x for w, x in zip(weights, d)) % 89 == 0+++def valid_tfn(value: str) -> bool:+ """9-digit TFN check: weights 1,4,3,7,5,8,6,9,10, sum mod 11 == 0.++ The ATO does not publish the TFN algorithm itself; the weights and modulus+ are the widely implemented ones, e.g. python-stdnum's ``stdnum.au.tfn``:+ https://arthurdejong.org/git/python-stdnum/tree/stdnum/au/tfn.py+ Older 8-digit TFNs are not recognised (no authoritative weights found).+ """+ d = _digits(value)+ if len(d) != 9:+ return False+ weights = (1, 4, 3, 7, 5, 8, 6, 9, 10)+ return sum(w * x for w, x in zip(weights, d)) % 11 == 0+++def valid_medicare(value: str) -> bool:+ """Medicare card number check (10 digits, issue number is the last).++ The first digit is 2-6; the first 8 digits are weighted 1,3,7,9,1,3,7,9,+ and the sum mod 10 must equal the 9th digit (the check digit); the 10th+ digit is the card issue number and is never 0. Algorithm as implemented by+ the ``starling`` package, which cites Services Australia:+ https://rdrr.io/cran/starling/src/R/check_medicare.R+ and described at https://doccy.com.au/medicare-number-validation+ """+ d = _digits(value)+ if len(d) != 10 or d[0] not in range(2, 7) or d[9] == 0:+ return False+ weights = (1, 3, 7, 9, 1, 3, 7, 9)+ return sum(w * x for w, x in zip(weights, d[:8])) % 10 == d[8]+++# --- patterns --------------------------------------------------------------++# A numeric identifier must stand alone: not glued to letters/digits, a hyphen+# ("ORD-123456782"), a currency sign, or a decimal/thousands separator.+_NUM_START = r"(?<![\w\-./$€£,])"+_NUM_END = r"(?![\w\-]|[.,]\d)"++# Email. ``\w`` is Unicode-aware, so internationalised mailboxes (RFC 6531)+# match. Scripts written without spaces (Han, kana, hangul) must not swallow the+# text beside the address, so the local part and each domain label are a run of+# either non-CJK word characters or CJK characters, never a mix. The last+# domain label is the TLD: letters only (at least two), or ``xn--`` punycode, so+# version specifiers such as ``[email protected]`` are not addresses.+_CJK = (+ "\u1100-\u11ff\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af"+ "\uf900-\ufaff\uff66-\uff9f"+)+_NOT_CJK = f"(?![{_CJK}])"+_NON_CJK_WORD_END = f"(?!{_NOT_CJK}[\\w\\-])"+_LOCAL = (+ r'(?:"[^"\\\r\n@]{1,64}"'+ + f"|(?:{_NOT_CJK}[\\w.%+\\-]){{1,64}}"+ + f"|[{_CJK}]{{1,64}})"+)+_LABEL = f"(?:(?:{_NOT_CJK}[^\\W_]|-){{1,63}}|[{_CJK}]{{1,63}})"+_TLD = (+ r"(?:xn--[A-Za-z0-9\-]{2,59}(?![\w\-])"+ + f"|(?:{_NOT_CJK}[^\\W\\d_]){{2,63}}{_NON_CJK_WORD_END}"+ + f"|[{_CJK}]{{2,63}}(?![{_CJK}]))"+)+_EMAIL_RE = re.compile(_LOCAL + "@(?:" + _LABEL + r"[.\u3002\uff0e\uff61]){1,8}" + _TLD)+# scp-style remote ``user@host:path``: the address is followed by ``:`` and a+# path, not by prose.+_SCP_PATH_RE = re.compile(r":(?:[/~]|[\w.\-]+/)")+++def _email_ok(m: Match[str]) -> bool:+ if m.group(0).partition("@")[0] == "git":+ return False # ssh user of every git host, never a person's mailbox+ return _SCP_PATH_RE.match(m.string, m.end()) is None+++_ABN_RE = re.compile(_NUM_START + r"\d{2}[ -]?\d{3}[ -]?\d{3}[ -]?\d{3}" + _NUM_END)+_TFN_RE = re.compile(_NUM_START + r"\d{3}[ -]?\d{3}[ -]?\d{3}" + _NUM_END)+_MEDICARE_RE = re.compile(_NUM_START + r"[2-6]\d{3}[ -]?\d{5}[ -]?\d" + _NUM_END)++_STREET_TYPES = (+ r"Street|St|Road|Rd|Avenue|Ave|Drive|Dr|Lane|Ln|Court|Ct|Place|Pl|Crescent|"+ r"Cres|Boulevard|Blvd|Terrace|Tce|Parade|Pde|Highway|Hwy|Way|Close|Circuit|"+ r"Cct|Esplanade|Esp|Grove|Square|Sq|Mews|Walk|Row"+)+_AU_STATES = r"NSW|VIC|QLD|SA|WA|TAS|NT|ACT"+# "12", "12A", "4/12", "Unit 4/12", "Flat 2, 12", "12-14"+_STREET_NO = (+ r"(?:(?:Unit|Apt|Apartment|Flat|Suite|Level|Lot)\s+\d{1,4}[A-Za-z]?[,/ ]\s*)?"+ r"\d{1,4}[A-Za-z]?(?:\s?[/-]\s?\d{1,4}[A-Za-z]?)?"+)+_SUBURB_TAIL = r"(?:,?\s+(?:[A-Za-z'\-]{2,20}\s+){0,3}(?:" + _AU_STATES + r")\s+\d{4})?"+# Capitalised street name: matches without further context ("12 Smith Street").+_ADDRESS_CAP_RE = re.compile(+ r"(?<![\w/])" + _STREET_NO + r"\s+(?:[A-Z][A-Za-z'’\-]{1,24}\s+){1,3}"+ r"(?:" + _STREET_TYPES + r")\b\.?" + _SUBURB_TAIL+)+# Any-case street name: only with a trailing "<suburb> <STATE> <postcode>", so+# "10 ways drive growth" and "5 best street food" are left alone.+_ADDRESS_CTX_RE = re.compile(+ r"(?<![\w/])" + _STREET_NO + r"\s+(?:[A-Za-z'’\-]{2,24}\s+){1,3}"+ r"(?:" + _STREET_TYPES + r")\b\.?,?\s+(?:[A-Za-z'\-]{2,20}\s+){0,3}"+ r"(?:" + _AU_STATES + r")\s+\d{4}\b",+ re.IGNORECASE,+)+_PO_BOX_RE = re.compile(+ r"\b(?:P\.?\s?O\.?\s+Box|GPO\s+Box|(?:Locked|Private)\s+Bag)\s+\d{1,6}\b",+ re.IGNORECASE,+)++# Phone numbers. International needs a leading "+"; Australian national forms+# are mobile 04xx xxx xxx and landline (0x) xxxx xxxx; NANP needs separators.+_PHONE_INTL_RE = re.compile(+ r"(?<![\w+\uff0b])[+\uff0b]\d{1,3}[ .\-]?(?:\(\d{1,4}\)[ .\-]?)?\d{1,4}"+ r"(?:[ .\-]?\d{2,5}){1,4}(?!\d)"+)+_PHONE_AU_RE = re.compile(+ r"(?<![\w\-./$+\uff0b])(?:\(0[2378]\)|0[2378])[ \-]?\d{4}[ \-]?\d{4}(?!\d)"+ r"|(?<![\w\-./$+\uff0b])(?:\(04\)|04)"+ r"(?:\d{2}[ .\-]?\d{3}[ .\-]?\d{3}|[ .\-]?\d{4}[ .\-]?\d{4})(?!\d)"+)+_PHONE_NANP_RE = re.compile(+ r"(?<![\w\-./$+\uff0b])(?:\(\d{3}\)[ ]?|\d{3}[ .\-])\d{3}[ .\-]\d{4}(?![\w\-]|[.,]\d)"+)+++# National significant number length by country code: +61 9 digits, +1 10,+# +44 10 (UK numbers are 9-10; the longer is taken). The trunk prefix after the+# country code ("(0)" or a leading 0 of +61/+44 numbers) is not part of it.+_NATIONAL_LEN = {"61": 9, "44": 10, "1": 10}+++def _intl_prefix_len(text: str) -> int:+ """Length of the phone-number prefix of an international-shaped match.++ The pattern is greedy, so digits that follow a number (a TFN, an order id)+ can be glued onto it. For a known country code the structure fixes how many+ national digits belong to the number (the trunk ``0`` is skipped) and+ exactly that many are consumed, even in the middle of a group. For other+ country codes the longest prefix of digit groups with at most 15 digits+ (E.164) and at least 8 is taken. 0 means no usable prefix.+ """+ digits = [(i, c) for i, c in enumerate(text) if c.isdigit()]+ num = "".join(c for _, c in digits)+ for cc in ("61", "44", "1"):+ if num.startswith(cc):+ national = num[len(cc) :]+ skip = 1 if cc != "1" and national.startswith("0") else 0+ need = _NATIONAL_LEN[cc]+ have = len(national) - skip+ if len(cc) + skip + min(have, need) < 8:+ return 0+ last = len(cc) + skip + min(have, need) - 1+ return digits[last][0] + 1+ groups = list(re.finditer(r"\d+", text))+ total = 0+ best = 0+ for g in groups:+ total += len(g.group(0))+ if total > 15:+ break+ if total >= 8:+ best = g.end()+ return best+++def _sub(+ text: str,+ pattern: "re.Pattern[str]",+ entity: str,+ found: List[str],+ accept: Callable[[Match[str]], bool] = lambda m: True,+) -> str:+ def repl(m: Match[str]) -> str:+ if not accept(m):+ return m.group(0)+ if entity not in found:+ found.append(entity)+ return _PLACEHOLDERS[entity]++ return pattern.sub(repl, text)+++def _sub_intl_phone(text: str, found: List[str]) -> str:+ def repl(m: Match[str]) -> str:+ raw = m.group(0)+ cut = _intl_prefix_len(raw)+ if cut == 0:+ return raw+ if PHONE not in found:+ found.append(PHONE)+ # Whatever follows the number stays for the later recognisers.+ return _PLACEHOLDERS[PHONE] + raw[cut:]++ return _PHONE_INTL_RE.sub(repl, text)+++def _spans(pattern: "re.Pattern[str]", text: str) -> List[Tuple[int, int]]:+ return [m.span() for m in pattern.finditer(text)]+++def redact_search_query(query: str) -> RedactionResult:+ """Return ``query`` with personal data replaced by typed placeholders.++ Never raises. On an internal failure the whole query is replaced by+ ``[REDACTED]`` (fail closed) and ``entities`` is ``("redaction_error",)``.+ """+ try:+ if not isinstance(query, str) or not query:+ return RedactionResult(query if isinstance(query, str) else "", ())+ found: List[str] = []+ text = query+ text = _sub(text, _EMAIL_RE, EMAIL, found, _email_ok)+ text = _sub(text, _ADDRESS_CAP_RE, ADDRESS, found)+ text = _sub(text, _ADDRESS_CTX_RE, ADDRESS, found)+ text = _sub(text, _PO_BOX_RE, ADDRESS, found)+ # "+" numbers first so their trailing digits are never read as a TFN.+ text = _sub_intl_phone(text, found)+ # Longest identifier first so an 11-digit ABN is never read as a TFN.+ text = _sub(text, _ABN_RE, ABN, found, lambda m: valid_abn(m.group(0)))+ text = _sub(+ text, _MEDICARE_RE, MEDICARE, found, lambda m: valid_medicare(m.group(0))+ )+ # What is still ABN- or Medicare-shaped here failed its checksum. A TFN+ # candidate that starts inside one is just its tail ("ABN 50 123 456 782"+ # must not become "ABN 50 [TFN]").+ rejected = _spans(_ABN_RE, text) + _spans(_MEDICARE_RE, text)+ text = _sub(+ text,+ _TFN_RE,+ TFN,+ found,+ lambda m: valid_tfn(m.group(0))+ and not any(a <= m.start() < b for a, b in rejected),+ )+ text = _sub(text, _PHONE_AU_RE, PHONE, found)+ text = _sub(text, _PHONE_NANP_RE, PHONE, found)+ return RedactionResult(text, tuple(found))+ except Exception:+ # Deliberately no query in the log line.+ logger.error("Search query redaction failed; query withheld", exc_info=False)+ return RedactionResult(_FAILED_PLACEHOLDER, (REDACTION_ERROR,))diff --git a/src/xagent/core/tools/core/tavily_web_search.py b/src/xagent/core/tools/core/tavily_web_search.pyindex af124289c..f05e66057 100644--- a/src/xagent/core/tools/core/tavily_web_search.py+++ b/src/xagent/core/tools/core/tavily_web_search.py@@ -9,6 +9,8 @@ from typing import Any, Dict, List, Optionalimport httpx+from .web_search_guard import run_guarded_search+logger = logging.getLogger(__name__)@@ -37,6 +39,23 @@ class TavilyWebSearchCore:Returns:List of search results with title, link, snippet and optionally content"""+ return await run_guarded_search(+ "tavily",+ query,+ lambda redacted: self._search_redacted(+ redacted, max_results, include_content, search_depth+ ),+ len,+ )++ async def _search_redacted(+ self,+ query: str,+ max_results: int,+ include_content: bool,+ search_depth: str,+ ) -> List[Dict[str, str]]:+ """Run the search; ``query`` has already been redacted."""logger.info("🔍 Tavily web search query='%s' max_results=%s depth=%s",query,diff --git a/src/xagent/core/tools/core/web_search.py b/src/xagent/core/tools/core/web_search.pyindex afed49663..b1518194a 100644--- a/src/xagent/core/tools/core/web_search.py+++ b/src/xagent/core/tools/core/web_search.py@@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optionalimport httpxfrom .web_content import WebContentFetcher, get_proxy_url+from .web_search_guard import run_guarded_searchlogger = logging.getLogger(__name__)@@ -30,6 +31,9 @@ class WebSearchCore:"""Search the web using Google Custom Search API.+ The query is redacted before it is sent or logged; see+ ``web_search_guard.run_guarded_search``.+Args:query: The search query stringnum_results: Number of results to return (max 10)@@ -38,9 +42,25 @@ class WebSearchCore:Returns:List of search results with title, link, snippet, and optional content"""+ return await run_guarded_search(+ "google",+ query,+ lambda redacted: self._search_redacted(+ redacted, num_results, include_content+ ),+ len,+ )++ async def _search_redacted(+ self,+ query: str,+ num_results: int,+ include_content: bool,+ ) -> List[Dict[str, str]]:+ """Run the search; ``query`` has already been redacted."""logger.info(- f"🔍 Starting web search for query: '{query}' "- f"(num_results={num_results}, include_content={include_content})"+ f"🔍 Starting web search (num_results={num_results}, "+ f"include_content={include_content}) query='{query}'")api_key = self._api_key or os.getenv("GOOGLE_API_KEY")diff --git a/src/xagent/core/tools/core/web_search_guard.py b/src/xagent/core/tools/core/web_search_guard.pynew file mode 100644index 000000000..3dc4c5aeb--- /dev/null+++ b/src/xagent/core/tools/core/web_search_guard.py@@ -0,0 +1,175 @@+"""Shared call step for every built-in web search provider.++Every provider core routes its ``search`` through :func:`run_guarded_search`:++1. the query is redacted (:mod:`.search_query_redaction`),+2. the provider is called with the redacted text only (if redaction itself+ fails, nothing is sent: the search fails with ``SearchQueryRedactionError``),+3. the optional audit hook is called with the redacted query.++The unredacted query exists only inside this function and the caller's+frame; neither the provider callable nor the audit hook ever receives it.++Audit hook+----------+Nothing is registered by default. A host (for example the SaaS layer, to write+one audit row per call) registers one callable with+:func:`set_search_audit_hook`. The callable receives a single+:class:`SearchAuditEvent` and may be sync or ``async``. An exception raised by+the hook is logged (without the query) and never fails the search.+"""++from __future__ import annotations++import asyncio+import inspect+import logging+from dataclasses import dataclass+from typing import Awaitable, Callable, Literal, Optional, Tuple, TypeVar, Union++from .search_query_redaction import REDACTION_ERROR, redact_search_query++logger = logging.getLogger(__name__)++T = TypeVar("T")+++@dataclass(frozen=True)+class SearchAuditEvent:+ """What the audit hook receives after each provider call."""++ # Provider name: "google", "tavily", "exa", "zhipu" (and "brave" once it+ # routes through the same step).+ provider: str+ # The query exactly as it was sent to the provider (already redacted).+ query: str+ # Entity types that were replaced, e.g. ("email", "phone"); () if none.+ entities: Tuple[str, ...]+ # Number of results the provider returned; None on failure.+ result_count: Optional[int]+ # "success" or "failure".+ outcome: Literal["success", "failure"]+ # On failure: the class name of the underlying error (the cause when the+ # core wrapped it, e.g. "ConnectTimeout", "HTTPStatusError"), "cancelled"+ # when the task was cancelled mid-call, or "redaction_error"; else None.+ failure_kind: Optional[str] = None+++class SearchQueryRedactionError(ValueError):+ """The query could not be redacted, so the search was not sent.++ The message deliberately carries no query text.+ """+++SearchAuditHook = Callable[[SearchAuditEvent], Union[None, Awaitable[None]]]++_audit_hook: Optional[SearchAuditHook] = None+++def set_search_audit_hook(hook: Optional[SearchAuditHook]) -> None:+ """Register the audit hook (replacing any previous one); ``None`` clears it."""+ global _audit_hook+ _audit_hook = hook+++def get_search_audit_hook() -> Optional[SearchAuditHook]:+ return _audit_hook+++def _failure_kind(exc: BaseException) -> str:+ return type(exc.__cause__ or exc).__name__+++async def _emit(event: SearchAuditEvent) -> None:+ hook = _audit_hook+ if hook is None:+ return+ try:+ result = hook(event)+ if inspect.isawaitable(result):+ await result+ except Exception as exc:+ # Log the failure class only: neither the hook's message nor the+ # event may carry anything we have not already redacted.+ logger.warning(+ "Search audit hook failed for provider %s: %s",+ event.provider,+ type(exc).__name__,+ )+++async def run_guarded_search(+ provider: str,+ query: str,+ call: Callable[[str], Awaitable[T]],+ count_results: Callable[[T], int],+) -> T:+ """Redact ``query``, run ``call`` with the redacted text, then audit.++ Args:+ provider: Provider name reported to the audit hook.+ query: The query as the model wrote it.+ call: The provider call; it receives the redacted query only.+ count_results: Maps the provider's return value to a result count.+ """+ redacted = redact_search_query(query)+ if REDACTION_ERROR in redacted.entities:+ # Fail closed: never fall back to the raw query, and never call the+ # provider with the placeholder either.+ error = SearchQueryRedactionError(+ "Web search was not sent because the query could not be redacted."+ )+ await _emit(+ SearchAuditEvent(+ provider=provider,+ query=redacted.text,+ entities=redacted.entities,+ result_count=None,+ outcome="failure",+ failure_kind=REDACTION_ERROR,+ )+ )+ raise error+ try:+ result = await call(redacted.text)+ except asyncio.CancelledError:+ # Not an ``Exception``: record the abandoned call, then let the+ # cancellation continue. The hook stays best-effort.+ await _emit(+ SearchAuditEvent(+ provider=provider,+ query=redacted.text,+ entities=redacted.entities,+ result_count=None,+ outcome="failure",+ failure_kind="cancelled",+ )+ )+ raise+ except Exception as exc:+ await _emit(+ SearchAuditEvent(+ provider=provider,+ query=redacted.text,+ entities=redacted.entities,+ result_count=None,+ outcome="failure",+ failure_kind=_failure_kind(exc),+ )+ )+ raise+ try:+ count: Optional[int] = count_results(result)+ except Exception:+ count = None+ await _emit(+ SearchAuditEvent(+ provider=provider,+ query=redacted.text,+ entities=redacted.entities,+ result_count=count,+ outcome="success",+ )+ )+ return resultdiff --git a/src/xagent/core/tools/core/zhipu_web_search.py b/src/xagent/core/tools/core/zhipu_web_search.pyindex 9e0a4c2a6..8ad88d48e 100644--- a/src/xagent/core/tools/core/zhipu_web_search.py+++ b/src/xagent/core/tools/core/zhipu_web_search.py@@ -11,6 +11,7 @@ import httpxfrom ...utils.public_http_transport import create_public_async_clientfrom .web_content import get_trusted_proxy_url+from .web_search_guard import run_guarded_searchlogger = logging.getLogger(__name__)@@ -51,6 +52,36 @@ class ZhipuWebSearchCore:Returns:Raw response JSON from Zhipu Web Search API"""+ return await run_guarded_search(+ "zhipu",+ query,+ lambda redacted: self._search_redacted(+ redacted,+ search_engine=search_engine,+ search_intent=search_intent,+ count=count,+ search_domain_filter=search_domain_filter,+ search_recency_filter=search_recency_filter,+ content_size=content_size,+ request_id=request_id,+ user_id=user_id,+ ),+ lambda response: len(response.get("search_result") or []),+ )++ async def _search_redacted(+ self,+ query: str,+ search_engine: str,+ search_intent: bool,+ count: int,+ search_domain_filter: Optional[str],+ search_recency_filter: str,+ content_size: str,+ request_id: Optional[str],+ user_id: Optional[str],+ ) -> Dict[str, Any]:+ """Run the search; ``query`` has already been redacted."""logger.info("🔍 Zhipu web search query='%s' engine=%s count=%s intent=%s",query,diff --git a/tests/core/tools/test_search_query_redaction.py b/tests/core/tools/test_search_query_redaction.pynew file mode 100644index 000000000..c4012aba5--- /dev/null+++ b/tests/core/tools/test_search_query_redaction.py@@ -0,0 +1,166 @@+"""Corpus tests for web search query redaction (true and false positives)."""++import pytest++from xagent.core.tools.core.search_query_redaction import (+ redact_search_query,+ valid_abn,+ valid_medicare,+ valid_tfn,+)+++ "query,expected,entities",+ [+ ("mail [email protected] now", "mail [EMAIL] now", ("email",)),+ ("[email protected]", "[EMAIL]", ("email",)),+ ("call +61 412 345 678", "call [PHONE]", ("phone",)),+ ("call 0412 345 678", "call [PHONE]", ("phone",)),+ ("call 0412345678", "call [PHONE]", ("phone",)),+ ("call (02) 9999 8888", "call [PHONE]", ("phone",)),+ ("call 02 9999 8888", "call [PHONE]", ("phone",)),+ ("call +44 20 7946 0958", "call [PHONE]", ("phone",)),+ ("call +1 (415) 555-0132", "call [PHONE]", ("phone",)),+ ("call 415-555-0132", "call [PHONE]", ("phone",)),+ ("lives at 12 Smith Street", "lives at [ADDRESS]", ("address",)),+ (+ "12 Smith Street, Surry Hills NSW 2010",+ "[ADDRESS]",+ ("address",),+ ),+ ("unit 4/12 george st sydney nsw 2000", "[ADDRESS]", ("address",)),+ ("Unit 4/12 George St", "[ADDRESS]", ("address",)),+ ("PO Box 1234", "[ADDRESS]", ("address",)),+ ("TFN 123 456 782", "TFN [TFN]", ("tfn",)),+ ("TFN 123456782", "TFN [TFN]", ("tfn",)),+ ("ABN 51 824 753 556", "ABN [ABN]", ("abn",)),+ ("ABN 51824753556", "ABN [ABN]", ("abn",)),+ ("Medicare 2123 45670 1", "Medicare [MEDICARE]", ("medicare",)),+ ("Medicare 2123456701", "Medicare [MEDICARE]", ("medicare",)),+ # digits glued behind a "+" number are not part of it (>15 digits)+ (+ "contact +61 412 345 678 123456782",+ "contact [PHONE] [TFN]",+ ("phone", "tfn"),+ ),+ (+ "call +44 20 7946 0958 then TFN 123 456 782",+ "call [PHONE] then TFN [TFN]",+ ("phone", "tfn"),+ ),+ # short trailing digits (a year) are not swallowed by a +61 number+ ("call +61 412 345 678 2024", "call [PHONE] 2024", ("phone",)),+ # an independent TFN next to an invalid ABN is still redacted+ (+ "TFN 123 456 782 ABN 50 123 456 782",+ "TFN [TFN] ABN 50 123 456 782",+ ("tfn",),+ ),+ (+ "[email protected] 0412 345 678",+ "[EMAIL] [PHONE]",+ ("email", "phone"),+ ),+ ],+)+def test_redacts_true_positives(query, expected, entities):+ result = redact_search_query(query)+ assert result.text == expected+ assert result.entities == entities+++ "query",+ [+ # years, dates, prices, versions+ "best laptops 2024",+ "released 2024-11-14 and 14/11/2024",+ "price $1,234.56 or 1234567 units",+ "python 3.12.1 release notes",+ "windows 11 22H2 build 22621.3007",+ # order numbers and product codes+ "order ORD-2024-118233 status",+ "invoice INV123456782",+ "ORD-123456782",+ "$123456782",+ "part number 4520-998877-X",+ # checksum failures survive+ "TFN 123 456 783",+ "ABN 51 824 753 557",+ # the tail of a failed ABN / Medicare is not a TFN+ "ABN 50 123 456 782",+ "Medicare 2123 456 782",+ "Medicare 2123 45671 1",+ "Medicare 2123456711",+ "number 98765432101",+ # names of people and organisations survive+ "Elon Musk Tesla Inc Sydney",+ "Jane Doe Acme Pty Ltd annual report",+ # street-type words without a house number or context+ "best street food in Sydney",+ "5 best street food places",+ "10 ways drive growth",+ "Smith Street Melbourne shops",+ "",+ ],+)+def test_leaves_false_positives_alone(query):+ result = redact_search_query(query)+ assert result.text == query+ assert result.entities == ()+++def test_checksums():+ assert valid_tfn("123 456 782") and not valid_tfn("123 456 783")+ assert valid_abn("51 824 753 556") and not valid_abn("51 824 753 557")+ assert valid_medicare("2123 45670 1") and not valid_medicare("2123 45671 1")+ # first digit outside 2-6 and issue number 0 are never Medicare numbers+ assert not valid_medicare("1123 45670 1")+ assert not valid_medicare("2123 45670 0")+ # wrong length+ assert not valid_tfn("12345678") and not valid_abn("1234567890")+++def test_fails_closed_without_raising(monkeypatch):+ import xagent.core.tools.core.search_query_redaction as mod++ def boom(*a, **k):+ raise RuntimeError("[email protected]")++ monkeypatch.setattr(mod, "_sub", boom)+ result = mod.redact_search_query("[email protected]")+ assert result.text == "[REDACTED]"+ assert result.entities == ("redaction_error",)+++def test_non_string_input_does_not_raise():+ assert redact_search_query(None).text == "" # type: ignore[arg-type]+++def test_long_input_is_fast():+ import time++ query = ("12 " + "a" * 40 + " ") * 2000+ start = time.perf_counter()+ redact_search_query(query)+ assert time.perf_counter() - start < 2.0+++ "hostile",+ [+ "é" * 60000,+ "a@" + "b." * 30000,+ "a-" * 40000 + "@x.y",+ "+61 " + "1 " * 30000,+ '"' + "a" * 60000,+ "例" * 40000 + "@" + "例" * 40000,+ ],+)+def test_hostile_unicode_input_stays_fast(hostile):+ import time++ start = time.monotonic()+ redact_search_query(hostile)+ assert time.monotonic() - start < 5diff --git a/tests/core/tools/test_search_query_redaction_matrix.py b/tests/core/tools/test_search_query_redaction_matrix.pynew file mode 100644index 000000000..f354508c5--- /dev/null+++ b/tests/core/tools/test_search_query_redaction_matrix.py@@ -0,0 +1,139 @@+"""Shape matrix for the redactor, one row per input form.++``FIXED`` rows are recognised and must come out exactly as listed.+``OUT_OF_SCOPE`` rows are the documented exclusions (see the scope statement in+``search_query_redaction``): they must come out unchanged, so a later change of+scope is a visible change in this file.+"""++import pytest++from xagent.core.tools.core.search_query_redaction import redact_search_query++FIXED = [+ ("alice@例子.公司", "[EMAIL]"),+ ("alice@münchen.de", "[EMAIL]"),+ ("[email protected]", "[EMAIL]"),+ ("josé.garcí[email protected]", "[EMAIL]"),+ ("用户@example.com", "[EMAIL]"),+ ("jane.doe@sub.例子.公司", "[EMAIL]"),+ ("jane@EXAMPLE。COM", "[EMAIL]"),+ ("[email protected]", "[EMAIL]"),+ ('"john doe"@example.com', "[EMAIL]"),+ ("mail [email protected].", "mail [EMAIL]."),+ ("([email protected])", "([EMAIL])"),+ ("<[email protected]>", "<[EMAIL]>"),+ ("mailto:[email protected]?subject=hi", "mailto:[EMAIL]?subject=hi"),+ ("[email protected]", "[EMAIL]"),+ ("[email protected],[email protected]", "[EMAIL],[EMAIL]"),+ # email inside text written without spaces: neighbours are not absorbed+ ("张三的邮箱[email protected]是否有效", "张三的邮箱[EMAIL]是否有效"),+ ("请联系[email protected]查询订单", "请联系[EMAIL]查询订单"),+ ("用户@example.com的资料", "[EMAIL]的资料"),+ ("ありがとう[email protected]です", "ありがとう[EMAIL]です"),+ ("한글[email protected]입니다", "한글[EMAIL]입니다"),+ ("see alice@例子.公司 for details", "see [EMAIL] for details"),+ # a CJK-only local part or label still absorbs adjacent CJK text+ ("联系用户@例子.公司查询", "[EMAIL]"),+ # documented false positive: alphabetic last part looks like a TLD+ ("[email protected]", "[EMAIL]"),+ ("[email protected]", "[EMAIL]"),+ ("[email protected]:note", "[EMAIL]:note"),+ # phone: international, trunk forms, separators+ ("call +61 (0) 412 345 678", "call [PHONE]"),+ ("call +61 (0)412 345 678", "call [PHONE]"),+ ("call +61 0412 345 678", "call [PHONE]"),+ ("call +61 (0)2 9999 8888", "call [PHONE]"),+ ("call +61 2 9999 8888", "call [PHONE]"),+ ("call +61412345678", "call [PHONE]"),+ ("call +61 412-345-678", "call [PHONE]"),+ ("call +61.412.345.678", "call [PHONE]"),+ ("call +61 412 345 678", "call [PHONE]"),+ ("tel:+61412345678", "tel:[PHONE]"),+ ("call +61 412 345 678123", "call [PHONE]123"),+ ("call +61 412 345 678 and 0412 345 678", "call [PHONE] and [PHONE]"),+ ("call +44 (0) 20 7946 0958", "call [PHONE]"),+ ("call +44 7911 123456", "call [PHONE]"),+ ("call +1 415 555 0132 x12", "call [PHONE] x12"),+ ("call 0412 345 678 ext 123", "call [PHONE] ext 123"),+ # phone: Australian national, NANP+ ("call 0412.345.678", "call [PHONE]"),+ ("call (04) 1234 5678", "call [PHONE]"),+ ("call 04 1234 5678", "call [PHONE]"),+ ("call 0412 345 678.", "call [PHONE]."),+ ("call 415.555.0132", "call [PHONE]"),+ ("call (415) 555-0132", "call [PHONE]"),+ # TFN / ABN / Medicare+ ("TFN 123 456 782", "TFN [TFN]"),+ ("TFN: 123-456-782", "TFN: [TFN]"),+ ("TFN 123 456 782.", "TFN [TFN]."),+ ("(123 456 782)", "([TFN])"),+ ("ABN 51 824 753 556", "ABN [ABN]"),+ ("ABN: 51-824-753-556", "ABN: [ABN]"),+ ("Medicare 2123 45670 1", "Medicare [MEDICARE]"),+ ("Medicare 2123-45670-1", "Medicare [MEDICARE]"),+ ("Medicare 2123456701/2", "Medicare [MEDICARE]/2"),+ ("Medicare 2123 45670 1 ref 1", "Medicare [MEDICARE] ref 1"),+ # addresses+ ("12 Smith Street", "[ADDRESS]"),+ ("12 Smith St.", "[ADDRESS]"),+ ("12a Smith Street", "[ADDRESS]"),+ ("Unit 4, 12 Smith Street", "[ADDRESS]"),+ ("4/12 Smith Street Surry Hills NSW 2010", "[ADDRESS]"),+ ("12 St Kilda Road Melbourne VIC 3004", "[ADDRESS]"),+ ("12 O'Connell Street", "[ADDRESS]"),+ ("Level 5, 100 Collins Street Melbourne VIC 3000", "[ADDRESS]"),+ ("Flat 2, 10 Downing Street", "[ADDRESS]"),+ ("PO Box 1234 Sydney NSW 2001", "[ADDRESS] Sydney NSW 2001"),+ ("GPO Box 99", "[ADDRESS]"),+ ("Locked Bag 5", "[ADDRESS]"),+ ("Private Bag 7", "[ADDRESS]"),+ # suburb and postcode alone are not personal data and stay+ ("12 Smith Street, Surry Hills, NSW, 2010", "[ADDRESS], Surry Hills, NSW, 2010"),+ ("12 Smith Street Surry Hills 2010", "[ADDRESS] Surry Hills 2010"),+ ("12 Smith Street Sydney", "[ADDRESS] Sydney"),+]++OUT_OF_SCOPE = [+ # email: the domain needs a dot; no address literals; no spaces around "@"+ "jane@localhost",+ "jane@[192.168.0.1]",+ "jane @ example.com",+ # email: version and package specifiers (numeric last label)+ "node package [email protected] changelog",+ "price list [email protected] release notes",+ # email: a domain label that mixes CJK and non-CJK characters+ "a@b例子.com",+ # email: scp-style remotes and the ssh user of git hosts+ "git ssh [email protected]:org/repo.git clone error",+ "[email protected]: Permission denied (publickey)",+ "[email protected]:/var/www",+ # phone: business numbers+ "call 1300 123 456",+ "call 13 12 34",+ # TFN / ABN: glued to letters, dot-separated, irregular grouping+ "tfn123456782",+ "TFN 123.456.782",+ "TFN 12 345 6782",+ "abn51824753556",+ "ABN 51.824.753.556",+ "company number 004 085 616",+ # addresses: lowercase without suburb context, non-English, not number-first+ "12 smith street",+ "12 Rue de la Paix Paris",+ "Hauptstraße 5 Berlin",+ "中山路 12 号",+]++[email protected]("query,expected", FIXED)+def test_matrix_fixed_rows(query, expected):+ assert redact_search_query(query).text == expected++[email protected]("query", OUT_OF_SCOPE)+def test_matrix_out_of_scope_rows_are_left_unchanged(query):+ result = redact_search_query(query)+ assert result.text == query+ assert result.entities == ()diff --git a/tests/core/tools/test_web_search_redaction_providers.py b/tests/core/tools/test_web_search_redaction_providers.pynew file mode 100644index 000000000..97ab86e17--- /dev/null+++ b/tests/core/tools/test_web_search_redaction_providers.py@@ -0,0 +1,729 @@+"""Every built-in web search provider sends only the redacted query.++HTTP uses a real ``httpx`` client on ``httpx.MockTransport``; the Exa SDK call+is stubbed. All personal data below is invented.+"""++import copy+import importlib+import json+import logging+import pkgutil+import sys+import types+from typing import Any, Dict, List++import httpx+import pytest++from xagent.core.agent.trace import (+ SEARCH_TOOL_NAMES,+ ConsoleTraceHandler,+ TraceAction,+ TraceCategory,+ TraceEvent,+ TraceEventType,+ TraceScope,+)+from xagent.core.tools.core import (+ tavily_web_search,+ web_search,+ zhipu_web_search,+)+from xagent.core.tools.core.exa_web_search import ExaWebSearchCore+from xagent.core.tools.core.tavily_web_search import TavilyWebSearchCore+from xagent.core.tools.core.web_search import WebSearchCore+from xagent.core.tools.core.web_search_guard import (+ SearchAuditEvent,+ get_search_audit_hook,+ set_search_audit_hook,+)+from xagent.core.tools.core.zhipu_web_search import ZhipuWebSearchCore++EMAIL = "[email protected]"+PHONE = "+61 412 345 678"+ADDRESS = "12 Smith Street, Surry Hills NSW 2010"+TFN = "123 456 782"+RAW_VALUES = [EMAIL, PHONE, ADDRESS, TFN, "412 345 678", "Smith Street", "123456782"]+QUERY = f"find {EMAIL} {PHONE} {ADDRESS} TFN {TFN} Acme Pty Ltd"+REDACTED = "find [EMAIL] [PHONE] [ADDRESS] TFN [TFN] Acme Pty Ltd"+EXPECTED_ENTITIES = ("email", "address", "tfn", "phone")++[email protected](autouse=True)+def _clean_hook(monkeypatch):+ monkeypatch.setenv("GOOGLE_API_KEY", "k")+ monkeypatch.setenv("GOOGLE_CSE_ID", "cx")+ monkeypatch.setenv("TAVILY_API_KEY", "k")+ monkeypatch.setenv("EXA_API_KEY", "k")+ monkeypatch.setenv("ZHIPU_API_KEY", "k")+ for var in ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"):+ monkeypatch.delenv(var, raising=False)+ set_search_audit_hook(None)+ yield+ set_search_audit_hook(None)+++class Wire:+ """Captures what the provider put on the wire."""++ def __init__(self) -> None:+ self.requests: List[httpx.Request] = []++ def text(self) -> str:+ return " ".join(f"{r.url} {r.content.decode()}" for r in self.requests)+++def _install_http(monkeypatch, module, payload: Dict[str, Any], status: int = 200):+ wire = Wire()++ def handler(request: httpx.Request) -> httpx.Response:+ wire.requests.append(request)+ return httpx.Response(status, json=payload)++ real_client = httpx.AsyncClient++ def factory(*args, **kwargs):+ kwargs.pop("proxy", None)+ return real_client(transport=httpx.MockTransport(handler), **kwargs)++ if module is zhipu_web_search:+ monkeypatch.setattr(+ module, "create_public_async_client", lambda **kw: factory()+ )+ else:+ monkeypatch.setattr(module.httpx, "AsyncClient", factory)+ return wire+++def _install_exa(monkeypatch):+ sent: List[Dict[str, Any]] = []++ class FakeExa:+ def __init__(self, api_key: str) -> None:+ self.headers: Dict[str, str] = {}++ def search(self, **kwargs: Any):+ sent.append(kwargs)+ return types.SimpleNamespace(+ results=[types.SimpleNamespace(title="T", url="https://e.com")]+ )++ monkeypatch.setitem(sys.modules, "exa_py", types.SimpleNamespace(Exa=FakeExa))+ return sent+++async def _run_provider(name: str, monkeypatch):+ """Run one provider with QUERY; return (text that reached the provider, results)."""+ if name == "google":+ wire = _install_http(+ monkeypatch,+ web_search,+ {"items": [{"title": "t", "link": "l", "snippet": "s"}]},+ )+ res = await WebSearchCore().search(QUERY)+ return wire.text(), res+ if name == "tavily":+ wire = _install_http(+ monkeypatch,+ tavily_web_search,+ {"results": [{"title": "t", "url": "u", "content": "c"}]},+ )+ res = await TavilyWebSearchCore().search(QUERY)+ return wire.text(), res+ if name == "zhipu":+ wire = _install_http(+ monkeypatch,+ zhipu_web_search,+ {"search_result": [{"title": "t", "link": "l", "content": "c"}]},+ )+ res = await ZhipuWebSearchCore().search(QUERY)+ return wire.text(), res+ sent = _install_exa(monkeypatch)+ res = await ExaWebSearchCore().search(QUERY)+ return json.dumps(sent), res+++PROVIDERS = ["google", "tavily", "zhipu", "exa"]+++def _decoded(text: str) -> str:+ from urllib.parse import unquote_plus++ return unquote_plus(text)++[email protected]("provider", PROVIDERS)+async def test_provider_receives_placeholders_only(provider, monkeypatch):+ wire_text, results = await _run_provider(provider, monkeypatch)+ wire_text = _decoded(wire_text)++ assert len(results["search_result"] if isinstance(results, dict) else results) == 1+ assert REDACTED in wire_text+ for value in RAW_VALUES:+ assert value not in wire_text+ assert "Acme Pty Ltd" in wire_text++[email protected]("provider", PROVIDERS)+async def test_info_logs_carry_no_raw_values(provider, monkeypatch, caplog):+ with caplog.at_level(logging.INFO):+ await _run_provider(provider, monkeypatch)++ logged = "\n".join(r.getMessage() for r in caplog.records)+ assert logged # the search did log something+ for value in RAW_VALUES:+ assert value not in logged+ assert "[EMAIL]" in logged # redacted text is what gets logged++[email protected]("provider", PROVIDERS)+async def test_audit_hook_receives_redacted_query(provider, monkeypatch):+ events: List[SearchAuditEvent] = []+ set_search_audit_hook(events.append)++ await _run_provider(provider, monkeypatch)++ assert len(events) == 1+ event = events[0]+ assert event.provider == provider+ assert event.query == REDACTED+ assert set(event.entities) == set(EXPECTED_ENTITIES)+ assert event.result_count == 1+ assert event.outcome == "success"+ assert event.failure_kind is None+ for value in RAW_VALUES:+ assert value not in repr(event)+++async def test_async_hook_is_awaited(monkeypatch):+ events: List[SearchAuditEvent] = []++ async def hook(event: SearchAuditEvent) -> None:+ events.append(event)++ set_search_audit_hook(hook)+ await _run_provider("tavily", monkeypatch)+ assert len(events) == 1++[email protected]("provider", PROVIDERS)+async def test_failing_hook_does_not_fail_search(provider, monkeypatch, caplog):+ def hook(event: SearchAuditEvent) -> None:+ raise RuntimeError(f"db down while writing {event.query} {EMAIL}")++ set_search_audit_hook(hook)+ with caplog.at_level(logging.INFO):+ _, results = await _run_provider(provider, monkeypatch)++ assert results+ logged = "\n".join(r.getMessage() for r in caplog.records)+ assert "audit hook failed" in logged+ assert EMAIL not in logged+ assert "[EMAIL]" not in logged.split("audit hook failed")[1]+++async def test_hook_reports_failure_outcome(monkeypatch):+ events: List[SearchAuditEvent] = []+ set_search_audit_hook(events.append)+ _install_http(monkeypatch, tavily_web_search, {"detail": "boom"}, status=500)++ with pytest.raises(ValueError):+ await TavilyWebSearchCore().search(QUERY)++ assert len(events) == 1+ assert events[0].outcome == "failure"+ assert events[0].failure_kind == "HTTPStatusError"+ assert events[0].result_count is None+ assert events[0].query == REDACTED+++async def test_cancelled_search_is_reported_and_stays_cancelled(monkeypatch):+ import asyncio++ events: List[SearchAuditEvent] = []+ set_search_audit_hook(events.append)+ started = asyncio.Event()++ async def slow(_q: str) -> str:+ started.set()+ await asyncio.sleep(30)+ return "never"++ from xagent.core.tools.core.web_search_guard import run_guarded_search++ task = asyncio.ensure_future(run_guarded_search("google", QUERY, slow, len))+ await started.wait()+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task++ assert task.cancelled()+ assert len(events) == 1+ assert events[0].outcome == "failure"+ assert events[0].failure_kind == "cancelled"+ assert events[0].result_count is None+ assert events[0].query == REDACTED+++async def test_no_hook_registered_by_default_and_search_unchanged(monkeypatch):+ assert get_search_audit_hook() is None+ _, results = await _run_provider("google", monkeypatch)+ assert results == [{"title": "t", "link": "l", "snippet": "s"}]+++async def test_query_without_personal_data_is_sent_unchanged(monkeypatch):+ wire = _install_http(monkeypatch, tavily_web_search, {"results": []})+ await TavilyWebSearchCore().search("Elon Musk Tesla 2024 price $1,234")+ assert json.loads(wire.requests[0].content)["query"] == (+ "Elon Musk Tesla 2024 price $1,234"+ )+++# --- console trace -----------------------------------------------------------++_SHAPES = [+ (TraceScope.ACTION, TraceAction.START),+ (TraceScope.ACTION, TraceAction.END),+ (TraceScope.ACTION, TraceAction.ERROR),+ # every console sink renders ``event.data``, whatever the scope+ (TraceScope.TASK, TraceAction.UPDATE),+ (TraceScope.STEP, TraceAction.UPDATE),+ (TraceScope.SYSTEM, TraceAction.UPDATE),+]+++ "tool_name", ["web_search", "zhipu_web_search", "exa_web_search"]+)[email protected]("scope,action", _SHAPES)+async def test_console_trace_shows_redacted_query(tool_name, scope, action, caplog):+ event = TraceEvent(+ TraceEventType(scope, action, TraceCategory.TOOL),+ task_id="t1",+ step_id="s1",+ data={+ "tool_name": tool_name,+ "tool_params": {"query": QUERY, "num_results": 3},+ "tool_call_id": "c1",+ },+ )+ with caplog.at_level(logging.INFO):+ await ConsoleTraceHandler().handle_event(event)++ logged = "\n".join(r.getMessage() for r in caplog.records)+ assert REDACTED in logged+ for value in RAW_VALUES:+ assert value not in logged+ # the stored payload is untouched+ assert event.data["tool_params"]["query"] == QUERY+++async def test_console_trace_leaves_other_tools_alone(caplog):+ event = TraceEvent(+ TraceEventType(TraceScope.ACTION, TraceAction.START, TraceCategory.TOOL),+ task_id="t1",+ step_id="s1",+ data={"tool_name": "write_file", "tool_params": {"query": EMAIL}},+ )+ with caplog.at_level(logging.INFO):+ await ConsoleTraceHandler().handle_event(event)+ assert EMAIL in "\n".join(r.getMessage() for r in caplog.records)+++# --- structural projection of search calls in console payloads ----------------++from xagent.core.agent.trace import _render_event_data_for_log # noqa: E402+++def _args(query: str = QUERY) -> Dict[str, Any]:+ return {"query": query, "num_results": 3}+++def _real_shapes(query: str = QUERY) -> Dict[str, Dict[str, Any]]:+ openai_call = {+ "id": "c1",+ "type": "function",+ "function": {"name": "web_search", "arguments": json.dumps(_args(query))},+ }+ return {+ # run_streaming_llm_call -> on_llm_end: normalised {name, args} calls+ "llm_end_args": {+ "response": {+ "content": "",+ "tool_calls": [+ {"id": "c1", "name": "web_search", "args": _args(query)}+ ],+ },+ "success": True,+ },+ # raw OpenAI shape: arguments is a JSON string+ "llm_end_openai": {+ "response": {"content": "", "tool_calls": [openai_call]},+ "success": True,+ },+ # the prompt replay in an LLM start event+ "context_preview": {+ "messages": [+ {"role": "user", "content": "hi"},+ {"role": "assistant", "content": None, "tool_calls": [openai_call]},+ ]+ },+ # checkpoint snapshot+ "checkpoint": {+ "pattern_state": {+ "pending_tool_calls": [+ {"id": "c1", "name": "web_search", "args": _args(query)}+ ],+ "tool_ledger": [+ {"tool_name": "exa_web_search", "tool_params": _args(query)}+ ],+ "last_response": {"tool_calls": [openai_call]},+ }+ },+ }++[email protected]("shape", list(_real_shapes()))[email protected]("max_bytes", [None, 0])+def test_console_render_redacts_real_payload_shapes(shape, max_bytes):+ data = _real_shapes()[shape]+ snapshot = copy.deepcopy(data)+ rendered = _render_event_data_for_log(data, max_bytes)+ for value in RAW_VALUES:+ assert value not in rendered+ assert "[EMAIL]" in rendered+ # the stored payload is untouched+ assert data == snapshot+++def test_console_render_redacts_unparseable_arguments_string():+ data = {+ "function": {+ "name": "web_search",+ "arguments": '{"query": "mail ' + EMAIL + ' now", "num_res',+ }+ }+ rendered = _render_event_data_for_log(data)+ assert EMAIL not in rendered and "mail" not in rendered+ assert "[search arguments withheld]" in rendered++[email protected]("shape", list(_real_shapes()))+def test_no_fragment_of_the_query_survives_any_byte_budget(shape):+ # Redaction happens before the byte budget cuts the payload, so no budget+ # can leave the start of an identifier behind.+ pad = "x " * 200+ data = _real_shapes(pad + QUERY)[shape]+ full = len(_render_event_data_for_log(data, 10**9))+ fragments = ["jane", "example.com", "412 3", "Smith", "123 4", "1234567"]+ for budget in range(40, full + 40, 13):+ rendered = _render_event_data_for_log(data, budget)+ for fragment in fragments:+ assert fragment not in rendered, (budget, fragment)+++def _replay_message(shape: str, query: str) -> Dict[str, Any]:+ arguments = json.dumps({"query": query, "n": 3})+ return {+ "openai_str": {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "c1",+ "type": "function",+ "function": {"name": "web_search", "arguments": arguments},+ }+ ],+ },+ "arguments_dict": {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "c1",+ "type": "function",+ "function": {"name": "web_search", "arguments": {"query": query}},+ }+ ],+ },+ # string arguments inside a dict that _shrink_message dumps again+ "legacy_function_call": {+ "role": "assistant",+ "content": None,+ "function_call": {"name": "web_search", "arguments": arguments},+ },+ "content_blocks": {+ "role": "assistant",+ "content": [+ {+ "type": "tool_use",+ "id": "c1",+ "name": "web_search",+ "input": {"query": query},+ }+ ],+ },+ "name_args": {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {"id": "c1", "name": "web_search", "args": {"query": query}}+ ],+ },+ }[shape]+++ "shape",+ [+ "openai_str",+ "arguments_dict",+ "legacy_function_call",+ "content_blocks",+ "name_args",+ ],+)[email protected]("pad", [0, 3, 8, 14])+def test_normalizer_then_console_never_leaves_a_query_fragment(shape, pad):+ # The trace boundary may cut a search call's arguments before the console+ # sink sees them (normalize_llm_trace_payload). Whatever budget cuts them,+ # the console line must carry no part of the identifier.+ from xagent.core.agent.trace import normalize_llm_trace_payload++ query = "word " * pad + "find [email protected] now"+ messages = [+ {"role": "user", "content": "hi " + "x" * 300},+ _replay_message(shape, query),+ ]+ fragments = ["jane", "@exa", "doe@", "example"]+ for budget in range(250, 1500, 3):+ stored = normalize_llm_trace_payload(+ {"context_preview": copy.deepcopy(messages), "success": True},+ max_bytes=budget,+ )+ snapshot = copy.deepcopy(stored)+ rendered = _render_event_data_for_log(stored)+ assert stored == snapshot # the stored payload is untouched+ for fragment in fragments:+ assert fragment not in rendered, (shape, pad, budget, fragment)+++def test_unparsed_arguments_fallback_input_is_redacted():+ # ReAct passes malformed JSON arguments on as {"input": <raw text>}.+ data = {+ "tool_name": "web_search",+ "tool_params": {"input": f"mail {EMAIL} or call {PHONE}"},+ }+ snapshot = copy.deepcopy(data)+ for max_bytes in (None, 0):+ rendered = _render_event_data_for_log(data, max_bytes)+ assert EMAIL not in rendered and "412 345" not in rendered+ assert "[EMAIL]" in rendered and "[PHONE]" in rendered+ assert data == snapshot+++def test_every_string_leaf_of_search_arguments_is_redacted():+ data = {+ "tool_name": "web_search",+ "tool_params": {+ "query": "ok",+ "filters": {"site": EMAIL, "tags": [f"a {PHONE}"]},+ "num_results": 3,+ },+ }+ rendered = _render_event_data_for_log(data)+ assert EMAIL not in rendered and "412 345" not in rendered+ assert "'num_results': 3" in rendered+++ "text",+ [+ '{"query": "find jane.doe@exa...[truncated 20 chars]',+ "{'query': 'find jane.doe@exa...[truncated 20 chars]",+ '[{"function_call": "{\\"query\\": \\"find jane.doe@exa...[truncated 9 chars]',+ '{\\\\"query\\\\" : \\\\"jane.doe@exa...[truncated 9 chars]',+ ],+)+def test_truncated_text_holding_a_query_key_is_withheld_whole(text):+ rendered = _render_event_data_for_log({"content": text})+ assert "jane" not in rendered and "@exa" not in rendered+ assert "[truncated text containing a query withheld]" in rendered+++def test_cut_leaf_inside_search_arguments_is_withheld():+ data = {+ "tool_name": "web_search",+ "tool_params": {"query": "find jane.doe@exa...[truncated 9 chars]"},+ }+ for max_bytes in (None, 0):+ rendered = _render_event_data_for_log(data, max_bytes)+ assert "jane" not in rendered and "@exa" not in rendered+ assert "[search arguments withheld]" in rendered+++def test_cut_query_text_is_withheld_when_nothing_is_truncated_by_the_sink():+ data = {"content": '{"query": "jane.doe@exa...[truncated 9 chars]'}+ rendered = _render_event_data_for_log(data, 0)+ assert "jane" not in rendered+ assert "[truncated text containing a query withheld]" in rendered+++def test_stringified_block_naming_a_search_tool_without_a_query_key():+ # the {"input": <raw text>} fallback inside a stringified content block+ complete = json.dumps(+ [{"type": "tool_use", "name": "web_search", "input": {"input": EMAIL}}]+ )+ rendered = _render_event_data_for_log({"content": complete})+ assert EMAIL not in rendered and "[EMAIL]" in rendered+ cut = complete[:-12] + "...[truncated 12 chars]"+ rendered = _render_event_data_for_log({"content": cut})+ assert "jane" not in rendered+ assert "[truncated text containing a query withheld]" in rendered+++def test_unquoted_search_tool_name_in_prose_is_kept():+ text = "I will use web_search for this...[truncated 40 chars]"+ assert text in _render_event_data_for_log({"content": text})+++def test_truncated_text_without_a_query_key_is_kept():+ text = "an ordinary long answer...[truncated 99 chars]"+ assert text in _render_event_data_for_log({"content": text})+++def test_truncation_marker_constant_matches_every_reducer():+ from xagent.core.agent import trace++ marker = trace._TRUNCATION_MARKER+ assert marker in trace._truncate_string("x" * 500, 50)+ assert marker in trace._trim_subtree("x" * 500, 50)+ assert marker in trace._shrink_node("x" * 500, [100], 1, set(), 12)+ middle = trace._reduce_text_list(["x" * 200] * 20, 400)+ assert any(isinstance(i, str) and marker in i for i in middle)+++def test_console_render_leaves_other_tools_alone():+ data = {"tool_calls": [{"name": "write_file", "args": {"query": EMAIL}}]}+ assert EMAIL in _render_event_data_for_log(data)+++def test_search_tool_name_set_covers_every_registered_search_tool():+ import xagent.core.tools.adapters.vibe as vibe++ names = set()+ for mod in pkgutil.iter_modules(vibe.__path__):+ if not mod.name.endswith("web_search"):+ continue+ module = importlib.import_module(f"{vibe.__name__}.{mod.name}")+ for obj in vars(module).values():+ if (+ isinstance(obj, type)+ and obj.__module__ == module.__name__+ and obj.__name__.endswith("Tool")+ ):+ names.add(obj.name.fget(None))+ assert names, "no search tool classes discovered"+ assert names <= SEARCH_TOOL_NAMES, names - SEARCH_TOOL_NAMES+++# --- redaction failure fails closed ------------------------------------------++[email protected]("provider", ["google", "tavily", "zhipu"])+async def test_redactor_failure_sends_nothing(provider, monkeypatch, caplog):+ import xagent.core.tools.core.search_query_redaction as redaction+ from xagent.core.tools.core.web_search_guard import SearchQueryRedactionError++ events: List[SearchAuditEvent] = []+ set_search_audit_hook(events.append)+ monkeypatch.setattr(+ redaction, "_sub", lambda *a, **k: (_ for _ in ()).throw(RuntimeError(EMAIL))+ )+ sent: List[httpx.Request] = []+ module = {+ "google": web_search,+ "tavily": tavily_web_search,+ "zhipu": zhipu_web_search,+ }[provider]+ wire = _install_http(monkeypatch, module, {})+ core = {+ "google": WebSearchCore,+ "tavily": TavilyWebSearchCore,+ "zhipu": ZhipuWebSearchCore,+ }[provider]()++ with caplog.at_level(logging.INFO):+ with pytest.raises(SearchQueryRedactionError) as info:+ await core.search(QUERY)++ assert wire.requests == [] and sent == []+ assert EMAIL not in str(info.value) and "Acme" not in str(info.value)+ assert len(events) == 1+ assert events[0].outcome == "failure"+ assert events[0].failure_kind == "redaction_error"+ assert events[0].query == "[REDACTED]"+ assert EMAIL not in "\n".join(r.getMessage() for r in caplog.records)+++async def test_redactor_failure_sends_nothing_exa(monkeypatch):+ import xagent.core.tools.core.search_query_redaction as redaction+ from xagent.core.tools.core.web_search_guard import SearchQueryRedactionError++ sent = _install_exa(monkeypatch)+ monkeypatch.setattr(+ redaction, "_sub", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("x"))+ )+ with pytest.raises(SearchQueryRedactionError):+ await ExaWebSearchCore().search(QUERY)+ assert sent == []+++# --- hook runs in the tool call's own context --------------------------------+++async def test_hook_sees_contextvars_of_the_search_call(monkeypatch):+ import contextvars++ marker: contextvars.ContextVar[str] = contextvars.ContextVar("marker")+ seen: List[str] = []++ async def hook(event: SearchAuditEvent) -> None:+ seen.append(f"async:{marker.get()}")++ def sync_hook(event: SearchAuditEvent) -> None:+ seen.append(f"sync:{marker.get()}")++ token = marker.set("task-42")+ try:+ set_search_audit_hook(hook)+ with monkeypatch.context() as m:+ await _run_provider("tavily", m)+ set_search_audit_hook(sync_hook)+ with monkeypatch.context() as m:+ await _run_provider("tavily", m)+ # a failing call reports from the same context too+ with monkeypatch.context() as m:+ _install_http(m, tavily_web_search, {}, status=500)+ with pytest.raises(ValueError):+ await TavilyWebSearchCore().search(QUERY)+ finally:+ marker.reset(token)++ assert seen == ["async:task-42", "sync:task-42", "sync:task-42"]