Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(platform): keep Google block data out of Langfuse traces and session replays (#15490)

Krzysztof Czerwinskiab3b36bf10f0
diff --git a/autogpt_platform/backend/backend/copilot/gate/held.py b/autogpt_platform/backend/backend/copilot/gate/held.py
index 5b47fc0eb5..d76e41f289 100644
--- a/autogpt_platform/backend/backend/copilot/gate/held.py
+++ b/autogpt_platform/backend/backend/copilot/gate/held.py
@@ -20,6 +20,11 @@ from pydantic import BaseModel, Field, ValidationError
from backend.copilot import woken_turns
from backend.copilot.constants import COPILOT_NODE_EXEC_ID_SEPARATOR
+from backend.copilot.google_data_redaction import (
+ is_google_call,
+ is_google_output,
+ remember_google_output,
+)
from backend.copilot.model import ChatSession
from backend.copilot.pending_messages import PendingMessage
from backend.copilot.tree import TreeRefusal, TurnEnvelope
@@ -340,13 +345,16 @@ async def _deliver(
user_id, session, call, get_tool(call.tool_name), card
)
shown = _card_copy(card)
+ google_data = is_google_call(call.args) or is_google_output(output)
try:
- return _result_row(call, cap(output), outcome, shown)
+ return _result_row(call, cap(output), outcome, shown, google_data)
except Exception:
# The outcome is known and may be a refusal; only the engine's cut
# failed, so deliver it trimmed rather than let recovery guess.
logger.warning(f"Could not cap held result {call.review_id}", exc_info=True)
- return _result_row(call, output[: _MAX_RESULT_CHARS // 2], outcome, shown)
+ return _result_row(
+ call, output[: _MAX_RESULT_CHARS // 2], outcome, shown, google_data
+ )
async def _recover(
@@ -378,13 +386,19 @@ def _result_row(
output: str,
outcome: Outcome,
shown: dict[str, Any] | None = None,
+ google_data: bool = False,
) -> PendingMessage:
+ content = (
+ f'<held_call_result tool="{call.tool_name}" '
+ f'tool_call_id="{call.tool_call_id}" review_id="{call.review_id}">\n'
+ f"{output}\n</held_call_result>"
+ )
+ # A user row, not a tool result, so the traces cannot tie it to a Google
+ # call, and its cut may no longer parse: remember it as delivered.
+ if google_data:
+ remember_google_output(content)
return HeldResult(
- content=(
- f'<held_call_result tool="{call.tool_name}" '
- f'tool_call_id="{call.tool_call_id}" review_id="{call.review_id}">\n'
- f"{output}\n</held_call_result>"
- ),
+ content=content,
metadata={
_RESULT_KEY: {
"review_id": call.review_id,
diff --git a/autogpt_platform/backend/backend/copilot/gate/held_google_data_test.py b/autogpt_platform/backend/backend/copilot/gate/held_google_data_test.py
new file mode 100644
index 0000000000..0b5d1fccd1
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/gate/held_google_data_test.py
@@ -0,0 +1,113 @@
+"""An approved held Google call's late result stays out of the traces."""
+
+import json
+from collections.abc import Callable
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from backend.blocks.google.gmail import GmailReadBlock
+from backend.blocks.text import MatchTextPatternBlock
+from backend.copilot.gate import held
+from backend.copilot.google_data_redaction import (
+ REDACTED,
+ redact_google_data,
+ remember_google_output,
+)
+from backend.copilot.model import ChatSession
+from backend.copilot.pending_message_helpers import combine_pending_with_current
+from backend.copilot.pending_messages import (
+ PendingMessage,
+ format_pending_as_user_message,
+)
+from backend.copilot.trial_cost_context import TrialCostContext
+
+EMAIL = "Subject: payroll -- the new salaries are attached"
+# Longer than the trimmed fallback keeps, so no cut of it parses.
+OUTPUT = json.dumps(
+ {"type": "block_output", "provider": "google", "outputs": EMAIL + "." * 70_000}
+)
+
+
+def _cut(text: str) -> str:
+ """A cap that leaves a preview which no longer parses."""
+ return f"<tool-output-truncated total_chars={len(text)}>\n{text[:200]}"
+
+
+def _failing(text: str) -> str:
+ raise RuntimeError("cap failed")
+
+
+async def _deliver(
+ monkeypatch: pytest.MonkeyPatch,
+ args: dict,
+ output: str,
+ cap: Callable[[str], str],
+) -> PendingMessage:
+ call = held.HeldCall(
+ review_id="review-1",
+ tool_name="run_block",
+ tool_call_id="call-1",
+ args=args,
+ cost_context=TrialCostContext(user_id="user-1", trial_id=None),
+ )
+ monkeypatch.setattr(held, "answered", AsyncMock(return_value=[call]))
+ monkeypatch.setattr(held, "_claim", AsyncMock(return_value=True))
+ monkeypatch.setattr(
+ held,
+ "review_db",
+ lambda: MagicMock(get_reviews_by_node_exec_ids=AsyncMock(return_value={})),
+ )
+ monkeypatch.setattr(held, "_outcome", AsyncMock(return_value=("approved", output)))
+ [row] = await held.resolve_answered(
+ "user-1", ChatSession.new(user_id="user-1", dry_run=False), cap=cap
+ )
+ return row
+
+
+def _traced(row: PendingMessage) -> str:
+ """The row as each engine hands it to the model, through the redaction:
+ a user row of its own (baseline) and folded into the turn's message (SDK)."""
+ folded = combine_pending_with_current([row], "go on", request_arrival_at=0)
+ return json.dumps(
+ redact_google_data(
+ [
+ format_pending_as_user_message(row),
+ {"role": "user", "content": [{"type": "text", "text": folded}]},
+ ]
+ )
+ )
+
+
[email protected]("cap", [_cut, _failing])
+async def test_a_held_google_call_result_is_redacted_however_it_was_cut(
+ monkeypatch: pytest.MonkeyPatch, cap: Callable[[str], str]
+):
+ args = {"block_id": GmailReadBlock().id, "input_data": {}}
+ row = await _deliver(monkeypatch, args, OUTPUT, cap)
+
+ assert "Subject: payroll" in row.content
+ traced = _traced(row)
+ assert "Subject: payroll" not in traced
+ assert REDACTED in traced
+
+
+async def test_a_held_call_whose_output_was_google_data_is_redacted(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """``continue_run_block`` names a review, not the block; ``BaseTool``
+ remembered the output the run produced."""
+ output = f"a preview of an email: {EMAIL}"
+ remember_google_output(output)
+ row = await _deliver(monkeypatch, {"review_id": "rev-0"}, output, _cut)
+
+ assert "Subject: payroll" not in _traced(row)
+
+
+async def test_a_held_call_of_another_block_stays_readable(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ args = {"block_id": MatchTextPatternBlock().id, "input_data": {}}
+ row = await _deliver(monkeypatch, args, "a harmless match", _cut)
+
+ assert "a harmless match" in _traced(row)
diff --git a/autogpt_platform/backend/backend/copilot/gate/reads_test.py b/autogpt_platform/backend/backend/copilot/gate/reads_test.py
index 987a5360b4..9f8cfd9718 100644
--- a/autogpt_platform/backend/backend/copilot/gate/reads_test.py
+++ b/autogpt_platform/backend/backend/copilot/gate/reads_test.py
@@ -1195,7 +1195,7 @@ async def test_a_declared_part_the_cap_cut_is_judged_as_cut(rows):
),
patch(
"backend.copilot.tools.base._persist_and_summarize",
- AsyncMock(side_effect=lambda raw_output, *args: raw_output),
+ AsyncMock(side_effect=lambda raw_output, *args, **kwargs: raw_output),
),
):
to_model = _text_from_mcp_result(await wrapper({"url": "u"}))
diff --git a/autogpt_platform/backend/backend/copilot/google_data_redaction.py b/autogpt_platform/backend/backend/copilot/google_data_redaction.py
new file mode 100644
index 0000000000..f5dee9a8eb
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/google_data_redaction.py
@@ -0,0 +1,384 @@
+"""Keep Google user data out of the LLM traces sent to Langfuse.
+
+Google's Limited Use policy bars staff from reading what the Gmail, Drive,
+Docs, Sheets and Calendar blocks return, so neither their output nor the
+arguments AutoPilot passed them may reach a trace. Every traced payload goes
+through :func:`redact_google_data`: the Langfuse SDK applies it as its
+``mask`` (baseline turns and every call through the Langfuse-wrapped OpenAI
+client), and LangSmith's Claude Agent SDK tracing, which exports SDK turns to
+Langfuse over OTLP, applies it as the client's ``hide_inputs`` and
+``hide_outputs``.
+
+Data counts as Google's by the block that produced it, never by what it says:
+
+* a call whose ``block_id`` (``run_block``), capability ``id``
+ (``run_capability``) or block ``review_id`` (``resume_capability``,
+ ``continue_run_block``) names a block on the ``google`` provider;
+* a ``block_output`` response whose ``provider`` is ``google``;
+* a tool output this process built from such a response, remembered by hash,
+ which covers outputs too large to hand the model whole (it then gets a
+ truncated preview that no longer parses);
+* the late result of an approved held call (``gate.held``), which reaches the
+ model as a ``<held_call_result>`` user row rather than as a tool result,
+ when the output it wraps is any of the above or the row was remembered.
+
+Within a conversation, the result answering a Google call is redacted whatever
+it says, and so is the input of a call whose result was Google data, so the
+history a later turn sends carries neither. Tool names, call ids, block ids,
+timings and token counts stay readable.
+
+The approval gate's model calls (the action supervisor and the content judge)
+quote a call's arguments or result inside a prompt of their own, where none of
+the above can find it, so a Google call's gate runs inside
+:func:`tracing_google_data`, which redacts every text those calls trace.
+"""
+
+import contextlib
+import functools
+import hashlib
+import json
+import re
+import threading
+from collections import OrderedDict
+from collections.abc import Iterator
+from contextvars import ContextVar
+from typing import Any
+
+import langsmith
+from langfuse import Langfuse
+from pydantic import BaseModel
+
+from backend.blocks import get_blocks
+from backend.copilot.capabilities.block_meta import get_block_provider
+from backend.copilot.constants import COPILOT_NODE_PREFIX, parse_node_id_from_exec_id
+from backend.integrations.providers import ProviderName
+from backend.util.settings import Settings
+
+REDACTED = "[redacted: Google data]"
+
+# Set on a ``tool-outputs/`` workspace file that saved a Google block's output,
+# so reading it back is recognised as Google data too.
+GOOGLE_DATA_METADATA_KEY = "google_data"
+
+# The arguments that carry what a call sends to the block; ``resume_capability``
+# re-runs it with ``input_overrides`` merged in.
+_PAYLOAD_KEYS = ("input_data", "input", "input_overrides")
+_REMEMBERED_LIMIT = 4096
+
+# How ``gate.held`` wraps a held call's late result.
+_HELD_RESULT_OPEN = re.compile(r"<held_call_result\b[^>]*>\n")
+_HELD_RESULT_CLOSE = "\n</held_call_result>"
+# Each pair hashes a slice of up to the whole text; a turn opens with a few
+# held results at most.
+_HELD_RESULT_PAIR_LIMIT = 100
+
+_remembered: OrderedDict[str, None] = OrderedDict()
+_remembered_lock = threading.Lock()
+
+_redacting_every_text: ContextVar[bool] = ContextVar(
+ "google_data_redacting_every_text", default=False
+)
+
+
+def redact_google_data(data: Any) -> Any:
+ """Return *data* with Google block calls and results replaced by
+ :data:`REDACTED`. Everything else is returned unchanged, unless
+ :func:`tracing_google_data` is active, when every text is replaced."""
+ if _redacting_every_text.get():
+ return _redact_every_text(data)
+ return _redact(data, _google_call_ids(data))
+
+
+def tracing_google_data(active: bool = True) -> Iterator[None]:
+ """While active, redact every text traced in this context, for model
+ calls handed a Google call's arguments or result as plain text."""
+ token = _redacting_every_text.set(_redacting_every_text.get() or active)
+ try:
+ yield
+ finally:
+ _redacting_every_text.reset(token)
+
+
+def mask_langfuse_data(*, data: Any, **kwargs: Any) -> Any:
+ """The Langfuse SDK's ``mask`` hook."""
+ return redact_google_data(data)
+
+
+def install_trace_redaction() -> None:
+ """Apply the redaction to both tracers.
+
+ Langfuse first: its client keeps the settings of the first one built in
+ the process, and building it sets the OpenTelemetry tracer provider the
+ LangSmith client then exports through, as it would have lazily.
+ """
+ install_langfuse_mask()
+ langsmith.configure(client=langsmith.Client(hide_inputs=_hide, hide_outputs=_hide))
+
+
+def install_langfuse_mask() -> None:
+ """Build the process's Langfuse client with the mask.
+
+ Has to run before anything calls ``langfuse.get_client()``, which hands
+ out copies of the first client built: the SDK keeps that one's settings
+ and drops a mask passed later. Raises when the client it gets back does
+ not mask, so the caller can leave tracing off rather than trace unmasked.
+ """
+ # The configured credentials, which ``.env`` puts in ``Settings`` but not
+ # in the environment the SDK would otherwise read them from. ``host``
+ # still yields to ``LANGFUSE_BASE_URL``, as it would by default.
+ secrets = Settings().secrets
+ client = Langfuse(
+ public_key=secrets.langfuse_public_key or None,
+ secret_key=secrets.langfuse_secret_key or None,
+ host=secrets.langfuse_host or None,
+ mask=mask_langfuse_data,
+ )
+ # ``_resources`` is None when the client is disabled (no keys): it sends
+ # nothing, so there is nothing to mask.
+ resources = client._resources
+ if resources is not None and resources.mask is not mask_langfuse_data:
+ raise RuntimeError(
+ "A Langfuse client was built before the Google data mask; "
+ "its traces would not be redacted"
+ )
+
+
+def remember_google_output(output: str) -> None:
+ """Record a tool output built from a Google block's result, so it is
+ redacted wherever it is traced even when it no longer parses."""
+ with _remembered_lock:
+ _remembered[_digest(output)] = None
+ while len(_remembered) > _REMEMBERED_LIMIT:
+ _remembered.popitem(last=False)
+
+
+def remember_cut_google_output(whole: str, cut: str) -> None:
+ """Remember *cut*, a later cut of the tool output *whole*, when *whole*
+ was Google data, so the text the model finally reads is redacted too."""
+ if cut != whole and _is_google_text(whole):
+ remember_google_output(cut)
+
+
+def is_google_output(text: str) -> bool:
+ """Whether the tool output *text* is Google data: a Google
+ ``block_output``, or an output remembered as built from one."""
+ digest = _digest(text)
+ with _remembered_lock:
+ if digest in _remembered:
+ _remembered.move_to_end(digest)
+ return True
+ if not text.startswith("{"):
+ return False
+ try:
+ parsed = json.loads(text)
+ except ValueError:
+ return False
+ return (
+ isinstance(parsed, dict)
+ and parsed.get("type") == "block_output"
+ and parsed.get("provider") == ProviderName.GOOGLE.value
+ )
+
+
+def is_google_call(arguments: dict[str, Any] | None) -> bool:
+ """Whether a tool call's arguments name a block on the ``google``
+ provider, as ``run_block``'s ``block_id``, ``run_capability``'s ``id``,
+ or the ``review_id`` ``resume_capability`` and ``continue_run_block``
+ take, which embeds the block id."""
+ if not arguments:
+ return False
+ return (
+ is_google_block_ref(arguments.get("block_id"))
+ or is_google_block_ref(arguments.get("id"))
+ or is_google_block_ref(_reviewed_block_id(arguments.get("review_id")))
+ )
+
+
+def is_google_block_ref(ref: object) -> bool:
+ """Whether *ref* names a block on the ``google`` provider, by id,
+ ``block:<id>`` capability id or class name."""
+ return isinstance(ref, str) and ref.strip().lower() in _google_block_refs()
+
+
+def _reviewed_block_id(review_id: object) -> str | None:
+ """The block a block review's id names: ``copilot-node-{block_id}:{hex}``."""
+ if not isinstance(review_id, str) or not review_id.startswith(COPILOT_NODE_PREFIX):
+ return None
+ return parse_node_id_from_exec_id(review_id).removeprefix(COPILOT_NODE_PREFIX)
+
+
+def _hide(data: dict) -> dict:
+ return redact_google_data(data)
+
+
+def _google_call_ids(data: Any) -> set[str]:
+ """Ids of the tool calls in *data* that ran a Google block: by the block
+ the call names, or by the Google data its result carries."""
+ ids: set[str] = set()
+ for node in _dicts(data):
+ call_id = node.get("id")
+ if isinstance(call_id, str) and is_google_call(_call_arguments(node)):
+ ids.add(call_id)
+ answered = _answered_call_id(node)
+ if answered is not None and _holds_google_output(node.get("content")):
+ ids.add(answered)
+ return ids
+
+
+def _redact(node: Any, google_ids: set[str]) -> Any:
+ node = _plain(node)
+ if isinstance(node, str):
+ return REDACTED if _is_google_text(node) else node
+ if isinstance(node, list):
+ return [_redact(item, google_ids) for item in node]
+ if not isinstance(node, dict):
+ return node
+ redacted = {key: _redact(value, google_ids) for key, value in node.items()}
+ if _answered_call_id(node) in google_ids and "content" in node:
+ redacted["content"] = REDACTED
+ call_id = node.get("id")
+ if isinstance(call_id, str) and call_id in google_ids:
+ _redact_call_arguments(redacted)
+ if is_google_call(node):
+ redacted = _without_payload(redacted)
+ return redacted
+
+
+def _redact_every_text(node: Any) -> Any:
+ node = _plain(node)
+ if isinstance(node, str):
+ return REDACTED
+ if isinstance(node, list):
+ return [_redact_every_text(item) for item in node]
+ if isinstance(node, dict):
+ return {key: _redact_every_text(value) for key, value in node.items()}
+ return node
+
+
+def _redact_call_arguments(call: dict[str, Any]) -> None:
+ """Redact the payload of a call already known to be Google's, in either
+ the Anthropic (``input``) or the OpenAI (``function.arguments``) shape."""
+ arguments = call.get("input")
+ if isinstance(arguments, dict):
+ call["input"] = _without_payload(arguments)
+ return
+ function = call.get("function")
+ if not isinstance(function, dict):
+ return
+ parsed = _call_arguments(call)
+ if parsed is not None:
+ call["function"] = {
+ **function,
+ "arguments": json.dumps(_without_payload(parsed)),
+ }
+
+
+def _without_payload(arguments: dict[str, Any]) -> dict[str, Any]:
+ return {
+ key: REDACTED if key in _PAYLOAD_KEYS else value
+ for key, value in arguments.items()
+ }
+
+
+def _call_arguments(node: dict[str, Any]) -> dict[str, Any] | None:
+ """The arguments of a tool call node, or None when *node* is not one."""
+ if node.get("type") == "tool_use":
+ arguments = node.get("input")
+ return arguments if isinstance(arguments, dict) else None
+ function = node.get("function")
+ if not isinstance(function, dict):
+ return None
+ raw = function.get("arguments")
+ if not isinstance(raw, str):
+ return None
+ try:
+ parsed = json.loads(raw)
+ except ValueError:
+ return None
+ return parsed if isinstance(parsed, dict) else None
+
+
+def _answered_call_id(node: dict[str, Any]) -> str | None:
+ """The call a tool result answers: Anthropic ``tool_use_id`` or OpenAI
+ (and LangSmith history) ``tool_call_id``."""
+ answered = node.get("tool_use_id") or node.get("tool_call_id")
+ return answered if isinstance(answered, str) else None
+
+
+def _holds_google_output(content: Any) -> bool:
+ if isinstance(content, str):
+ return _is_google_text(content)
+ if isinstance(content, list):
+ return any(_holds_google_output(item) for item in content)
+ if isinstance(content, dict):
+ return any(_holds_google_output(value) for value in content.values())
+ return False
+
+
+def _is_google_text(text: str) -> bool:
+ return is_google_output(text) or _wraps_google_held_result(text)
+
+
+def _wraps_google_held_result(text: str) -> bool:
+ """Whether *text* carries a held call's late result that is Google data.
+
+ The row may be folded into a longer message (the SDK's turn opener), and
+ the output it wraps may itself quote the closing tag, so every closing
+ tag after an opening one is tried. A text with more pairs than that
+ could take is redacted rather than scanned.
+ """
+ if "<held_call_result" not in text:
+ return False
+ closes = [
+ match.start() for match in re.finditer(re.escape(_HELD_RESULT_CLOSE), text)
+ ]
+ pairs = 0
+ for opening in _HELD_RESULT_OPEN.finditer(text):
+ for close in closes:
+ if close < opening.end():
+ continue
+ pairs += 1
+ if pairs > _HELD_RESULT_PAIR_LIMIT:
+ return True
+ row = text[opening.start() : close + len(_HELD_RESULT_CLOSE)]
+ inner = text[opening.end() : close]
+ if is_google_output(inner) or is_google_output(row):
+ return True
+ return False
+
+
+def _dicts(node: Any) -> Iterator[dict[str, Any]]:
+ node = _plain(node)
+ if isinstance(node, dict):
+ yield node
+ for value in node.values():
+ yield from _dicts(value)
+ elif isinstance(node, list):
+ for item in node:
+ yield from _dicts(item)
+
+
+def _plain(node: Any) -> Any:
+ """SDK message objects (the OpenAI client's tool calls) as plain data."""
+ return node.model_dump(mode="json") if isinstance(node, BaseModel) else node
+
+
+def _digest(text: str) -> str:
+ return hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest()
+
+
+def _google_block_refs() -> frozenset[str]:
+ refs: set[str] = set()
+ for block_id, block_class in get_blocks().items():
+ if get_block_provider(block_class()) != ProviderName.GOOGLE.value:
+ continue
+ refs.update(
+ (
+ block_id.lower(),
+ f"block:{block_id}".lower(),
+ block_class.__name__.lower(),
+ )
+ )
+ return frozenset(refs)
diff --git a/autogpt_platform/backend/backend/copilot/google_data_redaction_test.py b/autogpt_platform/backend/backend/copilot/google_data_redaction_test.py
new file mode 100644
index 0000000000..3fd2342df9
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/google_data_redaction_test.py
@@ -0,0 +1,465 @@
+"""What reaches Langfuse must not carry what a Google block returned."""
+
+import json
+from collections.abc import Iterator
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import langsmith
+import pytest
+from langfuse import Langfuse, get_client
+from langfuse._client.resource_manager import LangfuseResourceManager
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+from backend.blocks.google.gmail import GmailReadBlock
+from backend.blocks.text import MatchTextPatternBlock
+from backend.copilot.constants import COPILOT_NODE_PREFIX
+from backend.copilot.tools.models import BlockOutputResponse
+
+from . import google_data_redaction
+from .google_data_redaction import (
+ REDACTED,
+ install_langfuse_mask,
+ install_trace_redaction,
+ is_google_output,
+ mask_langfuse_data,
+ redact_google_data,
+ remember_google_output,
+ tracing_google_data,
+)
+
+GMAIL_BLOCK_ID = GmailReadBlock().id
+TEXT_BLOCK_ID = MatchTextPatternBlock().id
+EMAIL = "Subject: quarterly numbers -- revenue is down 12%"
+
+
+def _output(block_id: str, provider: str | None, text: str) -> str:
+ return BlockOutputResponse(
+ message="ran",
+ block_id=block_id,
+ block_name="block",
+ outputs={"result": [text]},
+ provider=provider,
+ ).model_dump_json(exclude_none=True)
+
+
+GMAIL_OUTPUT = _output(GMAIL_BLOCK_ID, "google", EMAIL)
+TEXT_OUTPUT = _output(TEXT_BLOCK_ID, None, "a harmless match")
+
+
+def _sdk_history(call: dict[str, Any], result: str) -> dict[str, Any]:
+ """An SDK turn's model input, as LangSmith records it."""
+ return {
+ "messages": [
+ {"role": "user", "content": "what did finance send me?"},
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "tool_use",
+ "id": "toolu_1",
+ "name": "mcp__copilot__run_block",
+ "input": call,
+ }
+ ],
+ },
+ {"role": "tool", "content": result, "tool_call_id": "toolu_1"},
+ ]
+ }
+
+
+def _baseline_history(call: dict[str, Any], result: str) -> list[dict[str, Any]]:
+ """A baseline turn's model input, as the Langfuse OpenAI wrapper records it."""
+ return [
+ {"role": "user", "content": "what did finance send me?"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "run_block", "arguments": json.dumps(call)},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call_1", "content": result},
+ ]
+
+
+def test_sdk_history_redacts_google_result_and_its_input():
+ call = {"block_id": GMAIL_BLOCK_ID, "input_data": {"query": "from:finance"}}
+
+ redacted = redact_google_data(_sdk_history(call, GMAIL_OUTPUT))
+
+ assistant, tool = redacted["messages"][1], redacted["messages"][2]
+ assert tool["content"] == REDACTED
+ assert tool["tool_call_id"] == "toolu_1"
+ tool_use = assistant["content"][0]
+ assert tool_use["name"] == "mcp__copilot__run_block"
+ assert tool_use["input"] == {"block_id": GMAIL_BLOCK_ID, "input_data": REDACTED}
+ assert EMAIL not in json.dumps(redacted)
+ assert "from:finance" not in json.dumps(redacted)
+
+
+def test_sdk_history_keeps_other_blocks_readable():
+ call = {"block_id": TEXT_BLOCK_ID, "input_data": {"text": "hello"}}
+ history = _sdk_history(call, TEXT_OUTPUT)
+
+ assert redact_google_data(history) == history
+
+
+def test_baseline_history_redacts_google_result_and_arguments():
+ call = {"id": f"block:{GMAIL_BLOCK_ID}", "input": {"query": "from:finance"}}
+
+ redacted = redact_google_data(_baseline_history(call, GMAIL_OUTPUT))
+
+ tool_call = redacted[1]["tool_calls"][0]
+ assert tool_call["function"]["name"] == "run_block"
+ assert json.loads(tool_call["function"]["arguments"]) == {
+ "id": f"block:{GMAIL_BLOCK_ID}",
+ "input": REDACTED,
+ }
+ assert redacted[2] == {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": REDACTED,
+ }
+
+
+def test_baseline_history_keeps_other_blocks_readable():
+ call = {"block_id": TEXT_BLOCK_ID, "input_data": {"text": "hello"}}
+ history = _baseline_history(call, TEXT_OUTPUT)
+
+ assert redact_google_data(history) == history
+
+
+def test_result_identifies_the_call_when_the_call_does_not():
+ """``continue_run_block`` names a review, not the block it resumes."""
+ call = {"review_id": "rev-1"}
+
+ redacted = redact_google_data(_baseline_history(call, GMAIL_OUTPUT))
+
+ assert redacted[2]["content"] == REDACTED
+ assert json.loads(redacted[1]["tool_calls"][0]["function"]["arguments"]) == call
+
+
+def test_google_call_without_its_result_redacts_the_result():
+ """A truncated result no longer parses; the call it answers still names
+ the block, as in history a later turn reloads."""
+ call = {"block_id": GMAIL_BLOCK_ID, "input_data": {}}
+ preview = '<tool-output-truncated total_chars=90000>\n{"type": "block_output", ...'
+
+ redacted = redact_google_data(_sdk_history(call, preview))
+
+ assert redacted["messages"][2]["content"] == REDACTED
+
+
+def test_tool_run_output_redacted_by_what_the_tool_produced():
+ """LangSmith records a tool run's output without its call."""
+ preview = f"<tool-output-truncated total_chars=90000>\n{EMAIL[:20]}"
+ remember_google_output(preview)
+
+ outputs = {"content": [{"type": "text", "text": preview}]}
+
+ assert redact_google_data(outputs) == {
+ "content": [{"type": "text", "text": REDACTED}]
+ }
+ assert redact_google_data({"output": GMAIL_OUTPUT}) == {"output": REDACTED}
+ assert redact_google_data({"output": TEXT_OUTPUT}) == {"output": TEXT_OUTPUT}
+
+
+def test_tool_run_input_redacted():
+ inputs = {"input": {"block_id": GMAIL_BLOCK_ID, "input_data": {"q": "x"}}}
+
+ assert redact_google_data(inputs) == {
+ "input": {"block_id": GMAIL_BLOCK_ID, "input_data": REDACTED}
+ }
+
+
+ "call",
+ [
+ # resume_capability
+ {
+ "review_id": f"{COPILOT_NODE_PREFIX}{GMAIL_BLOCK_ID}:1a2b3c4d",
+ "input_overrides": {"body": "the new email"},
+ },
+ # continue_run_block takes the review id alone; its input is the
+ # stored one, so only the id is left and stays readable.
+ {"review_id": f"{COPILOT_NODE_PREFIX}{GMAIL_BLOCK_ID}:1a2b3c4d"},
+ ],
+)
+def test_resumed_google_tool_run_input_redacted(call: dict[str, Any]):
+ """A LangSmith tool run records the call without its result; the review
+ id names the block it resumes."""
+ redacted = redact_google_data({"input": call})
+
+ assert "the new email" not in json.dumps(redacted)
+ assert redacted["input"]["review_id"] == call["review_id"]
+
+
+def test_resumed_tool_run_input_of_another_block_stays_readable():
+ inputs = {
+ "input": {
+ "review_id": f"{COPILOT_NODE_PREFIX}{TEXT_BLOCK_ID}:1a2b3c4d",
+ "input_overrides": {"text": "hello"},
+ }
+ }
+
+ assert redact_google_data(inputs) == inputs
+
+
+def test_google_block_output_response_is_remembered():
+ response = BlockOutputResponse(
+ message="ran", block_id=GMAIL_BLOCK_ID, block_name="b", outputs={}
+ )
+ assert not response.holds_google_data
+ assert response.model_copy(update={"provider": "google"}).holds_google_data
+
+
+def langfuse_spans() -> Iterator[InMemorySpanExporter]:
+ """A Langfuse client built the way production builds it, exporting to
+ memory instead of to Langfuse."""
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ LangfuseResourceManager.reset()
+ Langfuse(
+ public_key="pk-lf-test",
+ secret_key="sk-lf-test",
+ base_url="http://127.0.0.1:9",
+ tracer_provider=provider,
+ mask=mask_langfuse_data,
+ )
+ try:
+ yield exporter
+ finally:
+ LangfuseResourceManager.reset()
+
+
+def test_langfuse_generation_input_is_masked(langfuse_spans: InMemorySpanExporter):
+ call = {"block_id": GMAIL_BLOCK_ID, "input_data": {"query": "from:finance"}}
+ history = _baseline_history(call, GMAIL_OUTPUT)
+ client = get_client()
+
+ generation = client.start_observation(
+ name="OpenAI-generation", as_type="generation", input=history
+ )
+ generation.update(output={"role": "assistant", "content": "Done."})
+ generation.end()
+
+ sent = json.dumps(
+ [dict(span.attributes or {}) for span in langfuse_spans.get_finished_spans()]
+ )
+ assert EMAIL not in sent
+ assert "from:finance" not in sent
+ assert REDACTED in sent
+ assert "run_block" in sent
+
+
+def _build_langfuse(monkeypatch: pytest.MonkeyPatch, **kwargs: Any) -> None:
+ monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test")
+ monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test")
+ monkeypatch.setenv("LANGFUSE_BASE_URL", "http://127.0.0.1:9")
+ Langfuse(tracer_provider=TracerProvider(), **kwargs)
+
+
+def test_install_langfuse_mask_refuses_a_client_built_without_it(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """The SDK hands out the first client built; a mask passed later is
+ dropped, so installing it must fail rather than trace unmasked."""
+ LangfuseResourceManager.reset()
+ try:
+ _build_langfuse(monkeypatch)
+
+ with pytest.raises(RuntimeError):
+ install_langfuse_mask()
+ finally:
+ LangfuseResourceManager.reset()
+
+
+def test_install_langfuse_mask_accepts_a_client_built_with_it(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ LangfuseResourceManager.reset()
+ try:
+ _build_langfuse(monkeypatch, mask=mask_langfuse_data)
+
+ install_langfuse_mask()
+ finally:
+ LangfuseResourceManager.reset()
+
+
+def test_install_langfuse_mask_uses_the_configured_credentials():
+ """Credentials read from ``.env`` reach ``Settings`` but not the process
+ environment the SDK falls back to; without them the first client, which
+ every later ``get_client()`` reuses, would be disabled."""
+ secrets = MagicMock(
+ langfuse_public_key="pk-from-settings",
+ langfuse_secret_key="sk-from-settings",
+ langfuse_host="https://langfuse.example.com",
+ )
+ with (
+ patch.object(google_data_redaction, "Settings") as settings,
+ patch.object(google_data_redaction, "Langfuse") as langfuse,
+ ):
+ settings.return_value.secrets = secrets
+ langfuse.return_value._resources.mask = mask_langfuse_data
+ install_langfuse_mask()
+
+ langfuse.assert_called_once_with(
+ public_key="pk-from-settings",
+ secret_key="sk-from-settings",
+ host="https://langfuse.example.com",
+ mask=mask_langfuse_data,
+ )
+
+
+def test_install_trace_redaction_masks_both_tracers():
+ with (
+ patch.object(google_data_redaction, "Langfuse") as langfuse,
+ patch.object(google_data_redaction.langsmith, "configure") as configure,
+ ):
+ langfuse.return_value._resources.mask = mask_langfuse_data
+ install_trace_redaction()
+
+ langfuse.assert_called_once()
+ assert langfuse.call_args.kwargs["mask"] is mask_langfuse_data
+ client = configure.call_args.kwargs["client"]
+ assert isinstance(client, langsmith.Client)
+ inputs = {"input": {"block_id": GMAIL_BLOCK_ID, "input_data": {"q": "x"}}}
+ assert client._hide_run_inputs(inputs)["input"]["input_data"] == REDACTED
+ assert client._hide_run_outputs({"output": GMAIL_OUTPUT}) == {"output": REDACTED}
+
+
+def test_tracing_google_data_redacts_every_text(langfuse_spans: InMemorySpanExporter):
+ """A gate prompt quoting a Google result has no structure to find it by."""
+ prompt = [{"role": "user", "content": f"Judge this read:\n{EMAIL}"}]
+ client = get_client()
+
+ # An inner scope that is not a Google call keeps the outer one's redaction.
+ with tracing_google_data(), tracing_google_data(False):
+ generation = client.start_observation(
+ name="OpenAI-generation", as_type="generation", input=prompt
+ )
+ generation.update(output={"role": "assistant", "content": EMAIL})
+ generation.end()
+ assert redact_google_data(prompt) == prompt
+
+ sent = json.dumps(
+ [dict(span.attributes or {}) for span in langfuse_spans.get_finished_spans()]
+ )
+ assert "quarterly numbers" not in sent
+ assert REDACTED in sent
+
+
+def test_a_non_string_id_is_left_alone():
+ """Tool schemas carry ``{"id": {...}}``; it names no call."""
+ schema = {"properties": {"id": {"type": "string"}, "block_id": ["x"]}}
+
+ assert redact_google_data(schema) == schema
+
+
[email protected]("history", [_sdk_history, _baseline_history])
+def test_resumed_google_call_redacts_its_overrides(history):
+ """``resume_capability`` re-runs the block with ``input_overrides`` merged
+ into its input; its result is what identifies it as Google's."""
+ call = {"review_id": "rev-1", "input_overrides": {"body": "the new email"}}
+
+ sent = json.dumps(redact_google_data(history(call, GMAIL_OUTPUT)))
+
+ assert "the new email" not in sent
+ assert "rev-1" in sent
+
+
+def test_a_remembered_output_seen_again_outlives_newer_ones(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ monkeypatch.setattr(google_data_redaction, "_REMEMBERED_LIMIT", 2)
+ remember_google_output("first")
+ remember_google_output("second")
+
+ assert is_google_output("first")
+ remember_google_output("third")
+
+ assert is_google_output("first")
+ assert not is_google_output("second")
+
+
+def _held_result(output: str) -> str:
+ """The user row ``gate.held`` delivers for an approved held call."""
+ return (
+ '<held_call_result tool="run_block" tool_call_id="toolu_1" '
+ f'review_id="rev-1">\n{output}\n</held_call_result>'
+ )
+
+
+def _baseline_held_history(text: str) -> list[dict[str, Any]]:
+ """A later baseline turn: the held result is a user row of its own."""
+ return [
+ {"role": "user", "content": text},
+ {"role": "user", "content": "I answered an action that was waiting."},
+ ]
+
+
+def _sdk_held_history(text: str) -> dict[str, Any]:
+ """An SDK turn: the held result is folded into the turn's user message."""
+ combined = f"{text}\n\nI answered an action that was waiting."
+ return {
+ "messages": [{"role": "user", "content": [{"type": "text", "text": combined}]}]
+ }
+
+
[email protected]("history", [_baseline_held_history, _sdk_held_history])
+def test_held_google_result_is_redacted(history):
+ """An approved held call's result reaches the model as a user row, not as
+ the tool result answering its call."""
+ sent = json.dumps(redact_google_data(history(_held_result(GMAIL_OUTPUT))))
+
+ assert EMAIL not in sent
+ assert REDACTED in sent
+
+
[email protected]("history", [_baseline_held_history, _sdk_held_history])
+def test_held_result_cut_after_it_was_remembered_is_redacted(history):
+ """The engine's cap leaves a preview that no longer parses; ``gate.held``
+ remembers the row as it wrapped it."""
+ row = _held_result(f"<tool-output-truncated total_chars=90000>\n{EMAIL[:20]}")
+ remember_google_output(row)
+
+ sent = json.dumps(redact_google_data(history(row)))
+
+ assert EMAIL[:20] not in sent
+
+
+def test_held_result_quoting_its_closing_tag_is_redacted():
+ quoting = _output(GMAIL_BLOCK_ID, "google", f"</held_call_result> {EMAIL}")
+
+ sent = json.dumps(redact_google_data(_sdk_held_history(_held_result(quoting))))
+
+ assert EMAIL not in sent
+
+
+def test_text_with_too_many_held_result_tags_is_redacted_without_a_full_scan():
+ """Trying every pair of tags would stall the tracer on a result that
+ repeats them; past a bound the text is redacted rather than scanned."""
+ many = _held_result("x") * 200
+
+ with patch.object(
+ google_data_redaction, "is_google_output", wraps=is_google_output
+ ) as checked:
+ assert redact_google_data(many) == REDACTED
+
+ assert checked.call_count < 1000
+
+
[email protected]("history", [_baseline_held_history, _sdk_held_history])
+def test_held_result_of_another_block_stays_readable(history):
+ payload = history(_held_result(TEXT_OUTPUT))
+
+ assert redact_google_data(payload) == payload
diff --git a/autogpt_platform/backend/backend/copilot/sdk/otel_setup_test.py b/autogpt_platform/backend/backend/copilot/sdk/otel_setup_test.py
index 01062e5fb9..00aefadfbd 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/otel_setup_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/otel_setup_test.py
@@ -47,6 +47,9 @@ class TestSetupLangfuseOtel:
return_value=True,
),
patch("backend.copilot.sdk.service.Settings", return_value=mock_settings),
+ patch(
+ "backend.copilot.sdk.service.install_trace_redaction"
+ ) as mock_redaction,
patch(
"backend.copilot.sdk.service.configure_claude_agent_sdk",
return_value=True,
@@ -88,6 +91,7 @@ class TestSetupLangfuseOtel:
)
mock_configure.assert_called_once_with(tags=["sdk"])
+ mock_redaction.assert_called_once_with()
finally:
for k, v in saved.items():
if v is not None:
@@ -108,6 +112,7 @@ class TestSetupLangfuseOtel:
return_value=True,
),
patch("backend.copilot.sdk.service.Settings", return_value=mock_settings),
+ patch("backend.copilot.sdk.service.install_trace_redaction"),
patch(
"backend.copilot.sdk.service.configure_claude_agent_sdk",
return_value=True,
@@ -143,6 +148,81 @@ class TestSetupLangfuseOtel:
else:
os.environ.pop(key, None)
+ def test_sdk_left_untraced_when_redaction_cannot_install(self):
+ """Tracing without the Google data redaction would export it."""
+ mock_settings = MagicMock()
+ mock_settings.secrets.langfuse_public_key = "pk-test"
+ mock_settings.secrets.langfuse_secret_key = "sk-test"
+ mock_settings.secrets.langfuse_host = "https://langfuse.example.com"
+ mock_settings.secrets.langfuse_tracing_environment = "test"
+
+ with (
+ patch(
+ "backend.copilot.sdk.service._is_langfuse_configured",
+ return_value=True,
+ ),
+ patch("backend.copilot.sdk.service.Settings", return_value=mock_settings),
+ patch(
+ "backend.copilot.sdk.service.install_trace_redaction",
+ side_effect=RuntimeError("no client"),
+ ),
+ patch(
+ "backend.copilot.sdk.service.configure_claude_agent_sdk"
+ ) as mock_configure,
+ patch.dict(os.environ, {}, clear=False),
+ ):
+ from backend.copilot.sdk.service import _setup_langfuse_otel
+
+ _setup_langfuse_otel()
+
+ mock_configure.assert_not_called()
+
+ def test_tracing_left_off_when_redaction_cannot_install(self):
+ """LangSmith reads these to decide whether to trace at all, so they
+ must not outlive a redaction that failed to install."""
+ mock_settings = MagicMock()
+ mock_settings.secrets.langfuse_public_key = "pk-test"
+ mock_settings.secrets.langfuse_secret_key = "sk-test"
+ mock_settings.secrets.langfuse_host = "https://langfuse.example.com"
+ mock_settings.secrets.langfuse_tracing_environment = "test"
+ keys = [
+ "LANGSMITH_OTEL_ENABLED",
+ "LANGSMITH_OTEL_ONLY",
+ "LANGSMITH_TRACING",
+ "OTEL_EXPORTER_OTLP_HEADERS",
+ ]
+ seen_at_install: dict[str, str | None] = {}
+
+ def failing_install():
+ seen_at_install.update({key: os.environ.get(key) for key in keys})
+ raise RuntimeError("no client")
+
+ with (
+ patch(
+ "backend.copilot.sdk.service._is_langfuse_configured",
+ return_value=True,
+ ),
+ patch("backend.copilot.sdk.service.Settings", return_value=mock_settings),
+ patch(
+ "backend.copilot.sdk.service.install_trace_redaction",
+ side_effect=failing_install,
+ ),
+ patch("backend.copilot.sdk.service.configure_claude_agent_sdk"),
+ patch.dict(os.environ, {"OTEL_EXPORTER_OTLP_ENDPOINT": "https://mine"}),
+ ):
+ for key in keys:
+ os.environ.pop(key, None)
+ from backend.copilot.sdk.service import _setup_langfuse_otel
+
+ _setup_langfuse_otel()
+
+ # The LangSmith client picks its OTEL mode up as it is built.
+ assert seen_at_install["LANGSMITH_OTEL_ONLY"] == "true"
+ assert seen_at_install["LANGSMITH_TRACING"] is None
+ for key in keys:
+ assert key not in os.environ, f"{key} outlived the failed install"
+ assert os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] == "https://mine"
+
def test_graceful_failure_on_exception(self):
"""Setup should not raise even if internal code fails."""
with (
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 0d1101fdc5..5cbff68f21 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -57,6 +57,7 @@ from backend.copilot.budget_gate import gate_auto_continue
from backend.copilot.budget_signal import build_turn_budget_block
from backend.copilot.continuation import as_model_prompt
from backend.copilot.feedback_db import RATEABLE_ROLES
+from backend.copilot.google_data_redaction import install_trace_redaction
from backend.copilot.graphiti.context import fetch_warm_context
from backend.copilot.markers import append_error_marker
from backend.copilot.provider_failure import ProviderFailure
@@ -2012,47 +2013,63 @@ def _setup_langfuse_otel() -> None:
To route traces elsewhere, override `OTEL_EXPORTER_OTLP_ENDPOINT` and
`OTEL_EXPORTER_OTLP_HEADERS` environment variables — no code changes needed.
+
+ Every run is passed through the Google data redaction before it is
+ exported; if that cannot be installed, the SDK is left untraced.
"""
if not _is_langfuse_configured():
return
+ # Ours to take back if the setup fails, so LangSmith does not trace
+ # without the redaction.
+ set_here: list[str] = []
try:
settings = Settings()
pk = settings.secrets.langfuse_public_key
sk = settings.secrets.langfuse_secret_key
host = settings.secrets.langfuse_host
+ tracing_env = settings.secrets.langfuse_tracing_environment
# OTEL exporter config — these are only set if not already present,
- # so explicit env-var overrides always win.
+ # so explicit env-var overrides always win. The LangSmith client
+ # reads its OTEL mode as the redaction builds it, so these go first.
creds = base64.b64encode(f"{pk}:{sk}".encode()).decode()
- os.environ.setdefault("LANGSMITH_OTEL_ENABLED", "true")
- os.environ.setdefault("LANGSMITH_OTEL_ONLY", "true")
- os.environ.setdefault("LANGSMITH_TRACING", "true")
- os.environ.setdefault("OTEL_EXPORTER_OTLP_ENDPOINT", f"{host}/api/public/otel")
- os.environ.setdefault(
- "OTEL_EXPORTER_OTLP_HEADERS", f"Authorization=Basic {creds}"
- )
- os.environ.setdefault(
- "OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER",
- "autogpt-copilot-traces",
- )
-
- # Set the Langfuse environment via OTEL resource attributes so the
- # Langfuse server maps it to the first-class environment field.
- tracing_env = settings.secrets.langfuse_tracing_environment
- os.environ.setdefault(
- "OTEL_RESOURCE_ATTRIBUTES",
- f"langfuse.environment={tracing_env}",
+ set_here = _set_env_defaults(
+ {
+ "LANGSMITH_OTEL_ENABLED": "true",
+ "LANGSMITH_OTEL_ONLY": "true",
+ "OTEL_EXPORTER_OTLP_ENDPOINT": f"{host}/api/public/otel",
+ "OTEL_EXPORTER_OTLP_HEADERS": f"Authorization=Basic {creds}",
+ "OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER": (
+ "autogpt-copilot-traces"
+ ),
+ # The Langfuse server maps this to its first-class
+ # environment field.
+ "OTEL_RESOURCE_ATTRIBUTES": f"langfuse.environment={tracing_env}",
+ }
)
+ install_trace_redaction()
+ # Switched on only once every run it traces is redacted.
+ set_here += _set_env_defaults({"LANGSMITH_TRACING": "true"})
configure_claude_agent_sdk(tags=["sdk"])
logger.info(
"OTEL tracing configured for Claude Agent SDK → %s [%s]", host, tracing_env
)
except Exception:
+ for key in set_here:
+ os.environ.pop(key, None)
logger.warning("OTEL setup skipped — failed to configure", exc_info=True)
+def _set_env_defaults(defaults: dict[str, str]) -> list[str]:
+ """Set each variable not already set; return the ones this set."""
+ added = [key for key in defaults if key not in os.environ]
+ for key in added:
+ os.environ[key] = defaults[key]
+ return added
+
+
_setup_langfuse_otel()
diff --git a/autogpt_platform/backend/backend/copilot/sdk/tool_adapter.py b/autogpt_platform/backend/backend/copilot/sdk/tool_adapter.py
index 3816dc52c9..950a793d6a 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/tool_adapter.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/tool_adapter.py
@@ -38,6 +38,7 @@ from backend.copilot.gate.mcp_seam import (
screen_non_registry_read,
)
from backend.copilot.gate.reads import model_view
+from backend.copilot.google_data_redaction import remember_cut_google_output
from backend.copilot.model import ChatSession
from backend.copilot.sdk.file_ref import (
FileRefExpansionError,
@@ -863,6 +864,9 @@ def _make_truncating_wrapper(
result = await run(args)
truncated = truncate(result, _MCP_MAX_CHARS)
+ remember_cut_google_output(
+ _text_from_mcp_result(result), _text_from_mcp_result(truncated)
+ )
# Registry tools were judged inside ``BaseTool.execute`` on this cap.
if session is not None and name not in TOOL_REGISTRY:
truncated = await screen_non_registry_read(
diff --git a/autogpt_platform/backend/backend/copilot/sdk/tool_adapter_test.py b/autogpt_platform/backend/backend/copilot/sdk/tool_adapter_test.py
index 4f7cba1415..93cc7fcfc8 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/tool_adapter_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/tool_adapter_test.py
@@ -10,6 +10,11 @@ from mcp.types import ListToolsRequest, ToolAnnotations
from backend.copilot.builder_context import BUILDER_BLOCKED_TOOLS
from backend.copilot.context import get_sdk_cwd
+from backend.copilot.google_data_redaction import (
+ REDACTED,
+ redact_google_data,
+ remember_google_output,
+)
from backend.copilot.model import ChatSession
from backend.copilot.response_model import StreamToolOutputAvailable
from backend.copilot.tools import DEFERRED_TOOL_NAMES, TOOL_REGISTRY
@@ -1560,3 +1565,40 @@ class TestLateToolResultCap:
assert len(direct) < len(text)
assert cap_late_tool_result(text) == direct
+
+
+class TestCutGoogleOutputIsRemembered:
+ """A Google output too large for the model is cut here, after
+ ``BaseTool`` remembered it whole; the cut text has to be redacted too."""
+
+ @staticmethod
+ async def _cut(text: str) -> str:
+ async def handler(_args):
+ return {"content": [{"type": "text", "text": text}], "isError": False}
+
+ wrapper = _make_truncating_wrapper(handler, "google_cut_tool")
+ with patch(
+ "backend.copilot.sdk.tool_adapter.get_execution_context",
+ return_value=(None, None),
+ ):
+ result = await wrapper({"q": "x"})
+ cut = _text_from_mcp_result(result)
+ assert cut != text
+ return cut
+
+ @pytest.mark.asyncio
+ async def test_a_remembered_google_output(self):
+ text = "Subject: payroll " + "x" * (_MCP_MAX_CHARS + 5_000)
+ remember_google_output(text)
+
+ cut = await self._cut(text)
+
+ assert redact_google_data({"output": cut}) == {"output": REDACTED}
+
+ @pytest.mark.asyncio
+ async def test_other_output_stays_readable(self):
+ text = "a harmless page " + "y" * (_MCP_MAX_CHARS + 5_000)
+
+ cut = await self._cut(text)
+
+ assert redact_google_data({"output": cut}) == {"output": cut}
diff --git a/autogpt_platform/backend/backend/copilot/service.py b/autogpt_platform/backend/backend/copilot/service.py
index 9710de55a2..13575afcd9 100644
--- a/autogpt_platform/backend/backend/copilot/service.py
+++ b/autogpt_platform/backend/backend/copilot/service.py
@@ -36,6 +36,7 @@ from .config import ChatConfig, CopilotLLMModel
from .expert_context import OWNED_BLOCK_TAGS as _EXPERT_BLOCK_TAGS
from .expert_context import build_expert_context, escape_prompt_xml_tags
from .expert_kickoff import is_expert_kickoff_message
+from .google_data_redaction import install_langfuse_mask
from .llm_attribution import openrouter_attribution
from .model import (
ChatMessage,
@@ -437,6 +438,16 @@ def _is_langfuse_configured() -> bool:
)
+# Before anything asks for a Langfuse client: the SDK keeps the first one's
+# settings, and this one's mask keeps Google data out of the traces.
+if _is_langfuse_configured():
+ try:
+ install_langfuse_mask()
+ except Exception:
+ # The SDK path checks again and runs untraced; this is the alarm.
+ logger.exception("Langfuse traces are not masked for Google data")
+
+
async def _fetch_langfuse_prompt() -> str | None:
"""Fetch the static system prompt from Langfuse.
diff --git a/autogpt_platform/backend/backend/copilot/tools/base.py b/autogpt_platform/backend/backend/copilot/tools/base.py
index 616f71bbe5..b838d44329 100644
--- a/autogpt_platform/backend/backend/copilot/tools/base.py
+++ b/autogpt_platform/backend/backend/copilot/tools/base.py
@@ -9,6 +9,13 @@ from openai.types.chat import ChatCompletionToolParam
from backend.copilot.ai_spans import mark_tool_failed, mark_tool_raised, tool_span
from backend.copilot.context import get_current_envelope
+from backend.copilot.google_data_redaction import (
+ GOOGLE_DATA_METADATA_KEY,
+ is_google_call,
+ remember_cut_google_output,
+ remember_google_output,
+ tracing_google_data,
+)
from backend.copilot.model import ChatSession
from backend.copilot.response_model import StreamToolOutputAvailable
from backend.copilot.turn_tracing import tool_transaction
@@ -94,6 +101,7 @@ async def _persist_and_summarize(
session_id: str,
tool_call_id: str,
digest: bool = False,
+ google_data: bool = False,
) -> str:
"""Persist full output to workspace and return a preview with retrieval
instructions — a structural outline when *digest*, else a middle-out slice.
@@ -123,7 +131,10 @@ async def _persist_and_summarize(
filename=f"{tool_call_id}.json",
path=file_path,
mime_type="application/json",
- metadata={"purpose": "tool-output"},
+ metadata={
+ "purpose": "tool-output",
+ **({GOOGLE_DATA_METADATA_KEY: True} if google_data else {}),
+ },
overwrite=True,
)
except Exception:
@@ -536,7 +547,11 @@ class BaseTool:
# ``_execute_tool_sync`` — so there is one place to add, not two.
# Must stay AFTER the envelope and name gates: a call the envelope refuses can
# never run, so approving it would spend a user's decision on nothing.
- gated, decision = await self._gate(user_id, session, tool_call_id, kwargs)
+ # The gate's model calls quote the call's arguments and result in
+ # prompts of their own, so for a Google call their traces are redacted.
+ google_call = is_google_call(kwargs)
+ with tracing_google_data(google_call):
+ gated, decision = await self._gate(user_id, session, tool_call_id, kwargs)
if gated is not None:
return gated
run_kwargs = kwargs
@@ -567,14 +582,22 @@ class BaseTool:
threshold = _DIGEST_THRESHOLD if digest else _LARGE_OUTPUT_THRESHOLD
if len(raw_output) > threshold and user_id and session.session_id:
raw_output = await _persist_and_summarize(
- raw_output, user_id, session.session_id, tool_call_id, digest
+ raw_output,
+ user_id,
+ session.session_id,
+ tool_call_id,
+ digest,
+ google_data=result.holds_google_data,
)
-
output = StreamToolOutputAvailable(
toolCallId=tool_call_id,
toolName=self.name,
output=raw_output,
)
+ # Remembered as the model gets it, previews and caps included, so
+ # the traces can redact it even where it no longer parses.
+ if result.holds_google_data and isinstance(output.output, str):
+ remember_google_output(output.output)
except Exception as e:
logger.warning("Error in %s", self.name, exc_info=True)
tracing.set_status("internal_error")
@@ -589,9 +612,10 @@ class BaseTool:
).model_dump_json(),
success=False,
)
- return await self._screen_read(
- user_id, session, tool_call_id, kwargs, output, result.outside, full
- )
+ with tracing_google_data(google_call or result.holds_google_data):
+ return await self._screen_read(
+ user_id, session, tool_call_id, kwargs, output, result.outside, full
+ )
async def _gate(
self,
@@ -697,7 +721,8 @@ class BaseTool:
)
view = model_view.get()
if view is not None:
- seen = view(seen, result.success)
+ whole, seen = seen, view(seen, result.success)
+ remember_cut_google_output(whole, seen)
text, images = readable_parts(seen)
stub = await screen_read(
self.name,
diff --git a/autogpt_platform/backend/backend/copilot/tools/base_test.py b/autogpt_platform/backend/backend/copilot/tools/base_test.py
index e2968e06a3..767eb3ec6f 100644
--- a/autogpt_platform/backend/backend/copilot/tools/base_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/base_test.py
@@ -7,7 +7,16 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from backend.blocks.google.gmail import GmailReadBlock
+from backend.blocks.search import GetWikipediaSummaryBlock
from backend.copilot import ai_spans
+from backend.copilot.gate import ALLOW
+from backend.copilot.gate.reads import model_view
+from backend.copilot.google_data_redaction import (
+ GOOGLE_DATA_METADATA_KEY,
+ REDACTED,
+ redact_google_data,
+)
from backend.copilot.llm_attribution import set_turn_trace_id
from backend.copilot.model import ChatSession
from backend.copilot.tools.base import (
@@ -23,6 +32,7 @@ from backend.copilot.tools.base import (
from backend.copilot.tools.models import (
BlockDetails,
BlockDetailsResponse,
+ BlockOutputResponse,
ErrorResponse,
ResponseType,
ToolResponseBase,
@@ -820,3 +830,139 @@ class TestPostHogToolSpans:
assert properties["$ai_is_error"] is True
assert properties["$ai_error"] == "RuntimeError"
assert "inbox" not in repr(properties)
+
+
+# ---------------------------------------------------------------------------
+# The gate's traces of a Google call
+# ---------------------------------------------------------------------------
+
+
+class TestGateTracesOfGoogleCalls:
+ """The supervisor and the content judge quote a call's arguments and
+ result in prompts of their own, so for a Google call whatever they trace
+ is redacted whole."""
+
+ @staticmethod
+ def _block_tool(provider: str | None):
+ class _Block(BaseTool):
+ @property
+ def name(self) -> str:
+ return "run_capability"
+
+ @property
+ def description(self) -> str:
+ return "Runs a block"
+
+ @property
+ def parameters(self) -> dict:
+ return {"type": "object", "properties": {}}
+
+ async def _execute(self, user_id, session, **kwargs):
+ return BlockOutputResponse(
+ message="ran",
+ block_id="b",
+ block_name="b",
+ outputs={"result": ["the email"]},
+ provider=provider,
+ )
+
+ return _Block()
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "names_google_block, provider, action_redacted, read_redacted",
+ [
+ (True, None, True, True),
+ (False, "google", False, True),
+ (False, None, False, False),
+ ],
+ ids=["names a Google block", "returns Google data", "neither"],
+ )
+ async def test_redacted_while_the_gate_runs(
+ self, names_google_block, provider, action_redacted, read_redacted
+ ):
+ block = GmailReadBlock() if names_google_block else GetWikipediaSummaryBlock()
+ traced: dict[str, object] = {}
+
+ async def action(*args, **kwargs):
+ traced["action"] = redact_google_data({"text": "the email"})
+ return ALLOW
+
+ async def read(*args, **kwargs):
+ traced["read"] = redact_google_data({"text": "the email"})
+
+ with (
+ patch("backend.copilot.gate.check_action", side_effect=action),
+ patch(
+ "backend.copilot.gate.reads.release_held_read",
+ new=AsyncMock(return_value=None),
+ ),
+ patch("backend.copilot.gate.reads.screen_read", side_effect=read),
+ patch("backend.copilot.tools.base._record_activity", new=AsyncMock()),
+ ):
+ await self._block_tool(provider).execute(
+ "u1", MagicMock(session_id="s1"), "call-1", id=block.id
+ )
+
+ def expected(redacted: bool) -> dict[str, str]:
+ return {"text": REDACTED if redacted else "the email"}
+
+ assert traced == {
+ "action": expected(action_redacted),
+ "read": expected(read_redacted),
+ }
+ assert redact_google_data({"text": "the email"}) == expected(False)
+
+ @pytest.mark.asyncio
+ async def test_the_late_cut_of_a_google_output_is_redacted(self):
+ """The SDK's late path cuts the output again before the model reads it."""
+ seen: list[str] = []
+
+ def cut(text: str, success: bool) -> str:
+ seen.append(text[: len(text) // 2])
+ return seen[-1]
+
+ token = model_view.set(cut)
+ try:
+ with (
+ patch(
+ "backend.copilot.gate.check_action",
+ new=AsyncMock(return_value=ALLOW),
+ ),
+ patch(
+ "backend.copilot.gate.reads.release_held_read",
+ new=AsyncMock(return_value=None),
+ ),
+ patch(
+ "backend.copilot.gate.reads.screen_read",
+ new=AsyncMock(return_value=None),
+ ),
+ patch("backend.copilot.tools.base._record_activity", new=AsyncMock()),
+ ):
+ await self._block_tool("google").execute(
+ "u1", MagicMock(session_id="s1"), "call-1"
+ )
+ finally:
+ model_view.reset(token)
+
+ assert seen and redact_google_data({"o": seen[0]}) == {"o": REDACTED}
+
+
+class TestPersistedGoogleOutputIsFlagged:
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("google", [True, False])
+ async def test_the_saved_file_says_whether_it_holds_google_data(self, google):
+ workspace = MagicMock(id="ws-1")
+ db = AsyncMock()
+ db.get_or_create_workspace = AsyncMock(return_value=workspace)
+ manager = AsyncMock()
+ with (
+ patch("backend.copilot.tools.base.workspace_db", return_value=db),
+ patch("backend.copilot.tools.base.WorkspaceManager", return_value=manager),
+ ):
+ await _persist_and_summarize(
+ "A" * 200_000, "u1", "s1", "call-1", google_data=google
+ )
+
+ metadata = manager.write_file.await_args.kwargs["metadata"]
+ assert metadata.get(GOOGLE_DATA_METADATA_KEY, False) is google
diff --git a/autogpt_platform/backend/backend/copilot/tools/models.py b/autogpt_platform/backend/backend/copilot/tools/models.py
index d186698275..e4eddf433c 100644
--- a/autogpt_platform/backend/backend/copilot/tools/models.py
+++ b/autogpt_platform/backend/backend/copilot/tools/models.py
@@ -9,6 +9,7 @@ from pydantic import BaseModel, Field, PrivateAttr, model_validator
from backend.copilot.tools.execution_utils import NodeFailureSummary
from backend.data.graph import BaseGraph, GraphTriggerInfo
from backend.data.model import CredentialsMetaInput
+from backend.integrations.providers import ProviderName
class ResponseType(str, Enum):
@@ -175,6 +176,19 @@ class ToolResponseBase(BaseModel):
def outside(self) -> tuple[Any, ...] | None:
return self._outside
+ _google_data: bool = PrivateAttr(default=False)
+
+ def with_google_data(self, google_data: bool = True) -> Self:
+ """Mark this response as carrying what a Google block returned."""
+ self._google_data = self._google_data or google_data
+ return self
+
+ @property
+ def holds_google_data(self) -> bool:
+ """Whether this carries what a Google block returned, which must stay
+ out of LLM traces (see ``backend.copilot.google_data_redaction``)."""
+ return self._google_data
+
# Agent discovery models
class AgentInfo(BaseModel):
@@ -1026,6 +1040,10 @@ class BlockOutputResponse(ToolResponseBase):
None # only set to True on dry-run; omitted in normal runs
)
+ @property
+ def holds_google_data(self) -> bool:
+ return self.provider == ProviderName.GOOGLE.value or self._google_data
+
class ReviewRequiredResponse(ToolResponseBase):
"""Response when a block requires human review before execution."""
diff --git a/autogpt_platform/backend/backend/copilot/tools/workspace_files.py b/autogpt_platform/backend/backend/copilot/tools/workspace_files.py
index 0d30401442..3bdf5f529b 100644
--- a/autogpt_platform/backend/backend/copilot/tools/workspace_files.py
+++ b/autogpt_platform/backend/backend/copilot/tools/workspace_files.py
@@ -16,6 +16,7 @@ from backend.copilot.context import (
looks_like_sdk_tool_result_path,
sdk_tool_result_redirect_hint,
)
+from backend.copilot.google_data_redaction import GOOGLE_DATA_METADATA_KEY
from backend.copilot.model import ChatSession
from backend.copilot.tools.workdir import (
resolve_sandbox_path_or_error,
@@ -680,6 +681,9 @@ class ReadWorkspaceFileTool(BaseTool):
return resolved
target_file_id, file_info = resolved
+ # A saved Google block output, read back, is still Google data.
+ google_data = file_info.metadata.get(GOOGLE_DATA_METADATA_KEY) is True
+
# If save_to_path, read + save; cache bytes for possible inline reuse.
cached_content: bytes | None = None
if save_to_path:
@@ -715,7 +719,7 @@ class ReadWorkspaceFileTool(BaseTool):
f"from {file_info.name}"
),
session_id=session_id,
- )
+ ).with_google_data(google_data)
is_small = file_info.size_bytes <= self.MAX_INLINE_SIZE_BYTES
is_text = _is_text_mime(file_info.mime_type)
@@ -740,7 +744,7 @@ class ReadWorkspaceFileTool(BaseTool):
content_base64=base64.b64encode(content).decode("utf-8"),
message=msg,
session_id=session_id,
- )
+ ).with_google_data(google_data)
# Metadata + download URL for large/binary files
preview: str | None = None
@@ -775,7 +779,7 @@ class ReadWorkspaceFileTool(BaseTool):
preview=preview,
message=msg,
session_id=session_id,
- )
+ ).with_google_data(google_data)
except FileNotFoundError as e:
return ErrorResponse(message=str(e), session_id=session_id)
except WorkspaceAccessDeniedError as e:
diff --git a/autogpt_platform/backend/backend/copilot/tools/workspace_files_test.py b/autogpt_platform/backend/backend/copilot/tools/workspace_files_test.py
index 3bc91f6664..6e42a506de 100644
--- a/autogpt_platform/backend/backend/copilot/tools/workspace_files_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/workspace_files_test.py
@@ -3,11 +3,13 @@
import base64
import os
import shutil
-from unittest.mock import AsyncMock, patch
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from backend.copilot.context import SDK_PROJECTS_DIR, _current_project_dir
+from backend.copilot.google_data_redaction import GOOGLE_DATA_METADATA_KEY
from backend.copilot.tools._test_data import make_session, setup_test_data
from backend.copilot.tools.models import ErrorResponse
from backend.copilot.tools.workdir import validate_ephemeral_path
@@ -24,6 +26,7 @@ from backend.copilot.tools.workspace_files import (
_read_local_tool_result,
_resolve_write_content,
)
+from backend.data.workspace import WorkspaceFile
# Re-export so pytest discovers the session-scoped fixture
setup_test_data = setup_test_data
@@ -596,6 +599,9 @@ async def test_read_workspace_file_no_fallback_when_resolve_succeeds(setup_test_
mime_type = "text/plain"
size_bytes = len(fake_content)
+ def __init__(self) -> None:
+ self.metadata: dict = {}
+
mock_resolve = AsyncMock(return_value=(fake_file_id, _FakeFileInfo()))
mock_manager = AsyncMock()
@@ -950,3 +956,56 @@ class TestSkillsRegistryACL:
from backend.copilot.tools.workspace_files import _path_under_skills_registry
assert _path_under_skills_registry(path) is False
+
+
+class TestReadingASpilledGoogleOutput:
+ """A Google block's output too large for the model is saved to
+ ``tool-outputs/``; reading it back is Google data too."""
+
+ @staticmethod
+ async def _read(metadata: dict, **kwargs):
+ content = b'{"type": "block_output", "outputs": {"result": ["the email"]}}'
+ info = WorkspaceFile(
+ id="file-1",
+ workspace_id="ws-1",
+ created_at=datetime.now(UTC),
+ updated_at=datetime.now(UTC),
+ name="call-1.json",
+ path="/tool-outputs/call-1.json",
+ storage_path="s",
+ mime_type="application/json",
+ size_bytes=len(content) if kwargs.pop("small", True) else 10**6,
+ metadata=metadata,
+ )
+ manager = AsyncMock()
+ manager.read_file_by_id = AsyncMock(return_value=content)
+ with (
+ patch(
+ "backend.copilot.tools.workspace_files._resolve_file",
+ AsyncMock(return_value=(info.id, info)),
+ ),
+ patch(
+ "backend.copilot.tools.workspace_files.get_workspace_manager",
+ AsyncMock(return_value=manager),
+ ),
+ ):
+ return await ReadWorkspaceFileTool()._execute(
+ user_id="user-1",
+ session=MagicMock(session_id="s1"),
+ path="tool-outputs/call-1.json",
+ **kwargs,
+ )
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "read",
+ [{}, {"offset": 5, "length": 20}, {"small": False}],
+ ids=["whole", "a window", "a preview"],
+ )
+ async def test_flagged_as_google_data(self, read):
+ google = {"purpose": "tool-output", GOOGLE_DATA_METADATA_KEY: True}
+
+ assert (await self._read(google, **read)).holds_google_data
+ assert not (
+ await self._read({"purpose": "tool-output"}, **read)
+ ).holds_google_data
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ChainRowView.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ChainRowView.tsx
index da85c7951e..73a83c13b6 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ChainRowView.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/ChainRowView.tsx
@@ -3,6 +3,7 @@
import { ArrowDown01Icon } from "@hugeicons/core-free-icons";
import { useEffect, useRef, useState } from "react";
import { Icon } from "@/components/atoms/Icon/Icon";
+import { HIDDEN_FROM_REPLAY } from "@/lib/replay-privacy";
import { cn } from "@/lib/utils";
import { useCopilotUIStore } from "../../store";
import { ACCORDION_PANEL, accordionState, PANEL_REVEAL } from "./accordion";
@@ -12,6 +13,7 @@ import { useSubSessionEffectiveStatus } from "./SubSessionLive";
import { SwapText } from "./SwapText";
import { HeldTag } from "./HeldCallRowParts";
import { heldShowsOwnCall } from "./heldRow";
+import { hidesOutputFromReplay } from "./replayPrivacy";
import { getCatalogLabel } from "./toolCatalog";
import { ToolResult } from "./ToolResult";
import { ToolStatusBadge } from "./ToolStatusBadge";
@@ -214,7 +216,11 @@ export function ChainRowView({ row, isLast, readOnly = false }: Props) {
className="min-h-0 overflow-hidden"
>
<div
- className={cn("px-px pb-px pt-1.5", showContent && PANEL_REVEAL)}
+ className={cn(
+ "px-px pb-px pt-1.5",
+ showContent && PANEL_REVEAL,
+ hidesOutputFromReplay(row) && HIDDEN_FROM_REPLAY,
+ )}
>
{isReasoning ? (
<ReasoningStream
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/replayPrivacy.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/replayPrivacy.test.tsx
new file mode 100644
index 0000000000..22ef177fd5
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/__tests__/replayPrivacy.test.tsx
@@ -0,0 +1,145 @@
+import { render, screen } from "@/tests/integrations/test-utils";
+import { describe, expect, it } from "vitest";
+import { ChainRowView } from "../ChainRowView";
+import type { ChainRow } from "../helpers";
+
+const EMAIL = "Quarterly numbers: revenue is down 12%";
+
+function row(overrides: Partial<ChainRow>): ChainRow {
+ return {
+ key: "row-1",
+ category: "other",
+ text: "Ran Gmail Read",
+ state: "done",
+ ...overrides,
+ };
+}
+
+function gmailOutput() {
+ return {
+ type: "block_output",
+ message: "Block executed",
+ block_id: "25310c70-b89b-43ba-b25c-4dfa7e2a481c",
+ block_name: "Gmail Read",
+ provider: "google",
+ success: true,
+ outputs: { email: [EMAIL] },
+ };
+}
+
+function savedOutput() {
+ return {
+ type: "workspace_file_metadata",
+ message: "Read call-1.json",
+ file_id: "file-1",
+ name: "call-1.json",
+ path: "tool-outputs/call-1.json",
+ mime_type: "application/json",
+ size_bytes: 90000,
+ download_url: "/api/workspace/files/file-1/download",
+ preview: EMAIL,
+ };
+}
+
+function expectHiddenFromReplay(text: string) {
+ const element = screen.getByText(text, { exact: false });
+ expect(element.closest(".ph-no-capture")).not.toBeNull();
+ expect(element.closest(".sentry-block")).not.toBeNull();
+}
+
+describe("ChainRowView - session replay privacy", () => {
+ it("keeps a Google block's output out of session replays", () => {
+ render(
+ <ChainRowView
+ row={row({
+ tool: "run_block",
+ input: { block_id: gmailOutput().block_id, input_data: {} },
+ output: gmailOutput(),
+ })}
+ isLast
+ />,
+ );
+
+ expectHiddenFromReplay(EMAIL);
+ });
+
+ it("keeps a truncated block output out of session replays", () => {
+ const preview = `<tool-output-truncated total_chars=90000 workspace_path="tool-outputs/call-1.json">\n${EMAIL}\n</tool-output-truncated>`;
+ render(
+ <ChainRowView
+ row={row({
+ tool: "run_capability",
+ input: { id: "block:gmail", input: {} },
+ output: preview,
+ })}
+ isLast
+ />,
+ );
+
+ expectHiddenFromReplay(EMAIL);
+ });
+
+ it.each([
+ ["by its path", { path: "tool-outputs/call-1.json" }, savedOutput()],
+ // A large read reaches the row as a truncated preview, without a path.
+ [
+ "by its file id",
+ { file_id: "file-1" },
+ `<tool-output-truncated total_chars=90000>\n${EMAIL}`,
+ ],
+ ])(
+ "keeps a workspace file read %s out of session replays",
+ (_label, input, output) => {
+ render(
+ <ChainRowView
+ row={row({
+ tool: "read_workspace_file",
+ text: "Read call-1.json",
+ input,
+ output,
+ })}
+ isLast
+ />,
+ );
+
+ expectHiddenFromReplay(EMAIL);
+ },
+ );
+
+ it("keeps a workspace file read through run_capability out of session replays", () => {
+ render(
+ <ChainRowView
+ row={row({
+ tool: "run_capability",
+ text: "Read call-1.json",
+ input: {
+ id: "tool:read_workspace_file",
+ input: { path: "tool-outputs/call-1.json" },
+ },
+ output: savedOutput(),
+ })}
+ isLast
+ />,
+ );
+
+ expectHiddenFromReplay(EMAIL);
+ });
+
+ it("leaves other tools' output in session replays", () => {
+ render(
+ <ChainRowView
+ row={row({
+ tool: "search_docs",
+ text: "Searched docs",
+ input: { query: "agents" },
+ output: { result: "How to build an agent" },
+ })}
+ isLast
+ />,
+ );
+
+ const element = screen.getByText("How to build an agent", { exact: false });
+ expect(element.closest(".ph-no-capture")).toBeNull();
+ expect(element.closest(".sentry-block")).toBeNull();
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/replayPrivacy.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/replayPrivacy.ts
new file mode 100644
index 0000000000..e39011f8d7
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ToolChain/replayPrivacy.ts
@@ -0,0 +1,22 @@
+import { capabilityTargetRow } from "./capabilityRow";
+import type { ChainRow } from "./helpers";
+
+// Tools that hand back what a block returned: a Google block's output is the
+// user's Gmail, Drive, Docs, Sheets or Calendar data. Every block run is
+// hidden, not only Google's, because a large output reaches the chat as a
+// truncated preview that no longer reliably says which provider produced it.
+// `read_workspace_file` reads a large output back from `tool-outputs/`; every
+// read is hidden, because a read by `file_id` names no path, and a large read
+// reaches the chat as a truncated preview that no longer carries one.
+const HIDDEN_OUTPUT_TOOLS = new Set([
+ "run_block",
+ "continue_run_block",
+ "run_capability",
+ "resume_capability",
+ "read_workspace_file",
+]);
+
+export function hidesOutputFromReplay(row: ChainRow): boolean {
+ const tool = capabilityTargetRow(row).tool;
+ return tool !== undefined && HIDDEN_OUTPUT_TOOLS.has(tool);
+}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/tools/RunBlock/RunBlock.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/tools/RunBlock/RunBlock.tsx
index d26243cd72..27ed687b3f 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/tools/RunBlock/RunBlock.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/tools/RunBlock/RunBlock.tsx
@@ -1,6 +1,7 @@
"use client";
import { useGetV1ListAvailableBlocks } from "@/app/api/__generated__/endpoints/blocks/blocks";
+import { HIDDEN_FROM_REPLAY } from "@/lib/replay-privacy";
import type { ToolUIPart } from "ai";
import { MorphingTextAnimation } from "../../components/MorphingTextAnimation/MorphingTextAnimation";
import { ToolAccordion } from "../../components/ToolAccordion/ToolAccordion";
@@ -111,7 +112,10 @@ export function RunBlockTool({ part }: Props) {
)}
{hasExpandableContent && output && (
- <ToolAccordion {...getAccordionMeta(output)}>
+ <ToolAccordion
+ {...getAccordionMeta(output)}
+ className={HIDDEN_FROM_REPLAY}
+ >
{hasInputData && <BlockInputCard inputData={inputData} />}
{isRunBlockBlockOutput(output) && <BlockOutputCard output={output} />}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/tools/RunBlock/__tests__/RunBlock.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/tools/RunBlock/__tests__/RunBlock.test.tsx
index d060f4a167..27c8eed1ee 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/tools/RunBlock/__tests__/RunBlock.test.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/tools/RunBlock/__tests__/RunBlock.test.tsx
@@ -1,7 +1,7 @@
import { describe, expect, it } from "vitest";
import { getGetV1ListAvailableBlocksMockHandler } from "@/app/api/__generated__/endpoints/blocks/blocks.msw";
import { server } from "@/mocks/mock-server";
-import { render, screen } from "@/tests/integrations/test-utils";
+import { fireEvent, render, screen } from "@/tests/integrations/test-utils";
import { RunBlockTool, type RunBlockToolPart } from "../RunBlock";
const blockId = "87840993-2053-44b7-8da4-187ad4ee518c";
@@ -44,3 +44,31 @@ describe("RunBlockTool block name lookup", () => {
expect(screen.getByText('Running "My Block"')).not.toBeNull();
});
});
+
+describe("RunBlockTool session replay privacy", () => {
+ it("keeps the block's output out of session replays", async () => {
+ const email = "Quarterly numbers: revenue is down 12%";
+ render(
+ <RunBlockTool
+ part={makePart({
+ state: "output-available",
+ output: {
+ type: "block_output",
+ message: "Block executed",
+ block_id: blockId,
+ block_name: "Gmail Read",
+ provider: "google",
+ success: true,
+ outputs: { email: [email] },
+ },
+ })}
+ />,
+ );
+
+ fireEvent.click(screen.getByRole("button", { name: /Gmail Read/ }));
+
+ const output = await screen.findByText(email, { exact: false });
+ expect(output.closest(".ph-no-capture")).not.toBeNull();
+ expect(output.closest(".sentry-block")).not.toBeNull();
+ });
+});
diff --git a/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/__tests__/run-output-replay-privacy.test.tsx b/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/__tests__/run-output-replay-privacy.test.tsx
new file mode 100644
index 0000000000..3fb8f30dd2
--- /dev/null
+++ b/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/__tests__/run-output-replay-privacy.test.tsx
@@ -0,0 +1,144 @@
+import { getGetV2GetSpecificBlocksMockHandler } from "@/app/api/__generated__/endpoints/default/default.msw";
+import {
+ getGetV1GetExecutionDetailsMockHandler,
+ getGetV1ListGraphExecutionsMockHandler,
+} from "@/app/api/__generated__/endpoints/graphs/graphs.msw";
+import {
+ getGetV2GetLibraryAgentMockHandler,
+ getGetV2GetLibraryAgentResponseMock,
+} from "@/app/api/__generated__/endpoints/library/library.msw";
+import { getGetV2ListPresetsMockHandler } from "@/app/api/__generated__/endpoints/presets/presets.msw";
+import { getGetV1ListExecutionSchedulesForAGraphMockHandler } from "@/app/api/__generated__/endpoints/schedules/schedules.msw";
+import { AgentExecutionStatus } from "@/app/api/__generated__/models/agentExecutionStatus";
+import type { GraphExecutionMeta } from "@/app/api/__generated__/models/graphExecutionMeta";
+import { server } from "@/mocks/mock-server";
+import { render, screen } from "@/tests/integrations/test-utils";
+import { NuqsTestingAdapter } from "nuqs/adapters/testing";
+import { expect, test, vi } from "vitest";
+import { NewAgentLibraryView } from "../components/NewAgentLibraryView/NewAgentLibraryView";
+
+vi.mock("next/navigation", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("next/navigation")>()),
+ useParams: () => ({ id: "library-agent" }),
+ usePathname: () => "/library/agents/library-agent",
+ useRouter: () => ({ push: vi.fn(), replace: vi.fn(), refresh: vi.fn() }),
+ useSearchParams: () => new URLSearchParams(),
+}));
+
+vi.mock("@/lib/auth/hooks/useAuth", () => ({
+ useAuth: () => ({
+ user: { id: "user-1", email: "[email protected]" },
+ isLoggedIn: true,
+ isUserLoading: false,
+ }),
+}));
+
+vi.mock("@/services/feature-flags/use-get-flag", async (importOriginal) => ({
+ ...(await importOriginal<
+ typeof import("@/services/feature-flags/use-get-flag")
+ >()),
+ useGetFlag: () => false,
+ useFlagStatus: () => ({ enabled: false, ready: true }),
+}));
+
+const GRAPH_ID = "graph-1";
+const RUN_ID = "run-1";
+const EMAIL = "Quarterly numbers: revenue is down 12%";
+
+const GOOGLE_CREDENTIALS = {
+ type: "object",
+ properties: {
+ google_credentials: {
+ type: "object",
+ credentials_provider: ["google"],
+ credentials_types: ["oauth2"],
+ },
+ },
+};
+
+function makeRun(): GraphExecutionMeta {
+ return {
+ id: RUN_ID,
+ user_id: "user-1",
+ graph_id: GRAPH_ID,
+ graph_version: 1,
+ inputs: {},
+ credential_inputs: {},
+ nodes_input_masks: {},
+ preset_id: null,
+ status: AgentExecutionStatus.COMPLETED,
+ started_at: new Date(),
+ ended_at: new Date(),
+ stats: { node_exec_count: 1, node_error_count: 0 },
+ };
+}
+
+function renderRun(credentialsInputSchema: Record<string, unknown>) {
+ const run = makeRun();
+ server.use(
+ getGetV2GetLibraryAgentMockHandler(
+ getGetV2GetLibraryAgentResponseMock({
+ id: "library-agent",
+ graph_id: GRAPH_ID,
+ graph_version: 1,
+ name: "Inbox digest",
+ input_schema: { type: "object", properties: {} },
+ credentials_input_schema: credentialsInputSchema,
+ has_external_trigger: false,
+ trigger_setup_info: null,
+ has_sensitive_action: false,
+ has_human_in_the_loop: false,
+ recommended_schedule_cron: null,
+ }),
+ ),
+ getGetV1ListGraphExecutionsMockHandler({
+ executions: [run],
+ pagination: {
+ total_items: 1,
+ total_pages: 1,
+ current_page: 1,
+ page_size: 20,
+ },
+ }),
+ getGetV1ListExecutionSchedulesForAGraphMockHandler([]),
+ getGetV2ListPresetsMockHandler({
+ presets: [],
+ pagination: {
+ total_items: 0,
+ total_pages: 0,
+ current_page: 1,
+ page_size: 100,
+ },
+ }),
+ getGetV1GetExecutionDetailsMockHandler({
+ ...run,
+ inputs: {},
+ outputs: { digest: [EMAIL] },
+ node_executions: [],
+ }),
+ getGetV2GetSpecificBlocksMockHandler([]),
+ );
+
+ render(
+ <NuqsTestingAdapter searchParams={`activeTab=runs&activeItem=${RUN_ID}`}>
+ <NewAgentLibraryView />
+ </NuqsTestingAdapter>,
+ );
+}
+
+// Every workflow's output is hidden, not only one listing a Google account: a
+// Google Drive picker's credential is left out of credentials_input_schema, so
+// a workflow reading a picked spreadsheet lists none.
+test.each([
+ ["lists a Google account", GOOGLE_CREDENTIALS],
+ ["lists no Google account", { type: "object", properties: {} }],
+])(
+ "a run output is kept out of session replays when the workflow %s",
+ async (_, credentialsInputSchema) => {
+ renderRun(credentialsInputSchema);
+
+ const output = await screen.findByText(EMAIL);
+ expect(output.closest(".ph-no-capture")).not.toBeNull();
+ expect(output.closest(".sentry-block")).not.toBeNull();
+ },
+);
diff --git a/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/selected-views/SelectedRunView/SelectedRunView.tsx b/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/selected-views/SelectedRunView/SelectedRunView.tsx
index 0759e5efaf..71faa443ea 100644
--- a/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/selected-views/SelectedRunView/SelectedRunView.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/selected-views/SelectedRunView/SelectedRunView.tsx
@@ -182,7 +182,10 @@ export function SelectedRunView({
<LoadingSpinner />
</div>
) : run && "outputs" in run ? (
- <RunOutputs outputs={run.outputs as any} />
+ <RunOutputs
+ outputs={run.outputs as any}
+ hiddenFromReplay
+ />
) : (
<Text variant="body" className="text-neutral-600">
No output from this run.
diff --git a/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/selected-views/SelectedRunView/components/RunOutputs.tsx b/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/selected-views/SelectedRunView/components/RunOutputs.tsx
index deff4a3c03..19ae7da650 100644
--- a/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/selected-views/SelectedRunView/components/RunOutputs.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/selected-views/SelectedRunView/components/RunOutputs.tsx
@@ -9,6 +9,8 @@ import {
OutputActions,
OutputItem,
} from "@/components/contextual/OutputRenderers";
+import { HIDDEN_FROM_REPLAY } from "@/lib/replay-privacy";
+import { cn } from "@/lib/utils";
import React, { useMemo } from "react";
type OutputsRecord = Record<string, Array<unknown>>;
@@ -16,9 +18,14 @@ type OutputsRecord = Record<string, Array<unknown>>;
interface RunOutputsProps {
outputs: OutputsRecord;
shareToken?: string;
+ hiddenFromReplay?: boolean;
}
-export function RunOutputs({ outputs, shareToken }: RunOutputsProps) {
+export function RunOutputs({
+ outputs,
+ shareToken,
+ hiddenFromReplay = false,
+}: RunOutputsProps) {
const items = useMemo(() => {
const list: Array<{
key: string;
@@ -85,7 +92,12 @@ export function RunOutputs({ outputs, shareToken }: RunOutputsProps) {
}
return (
- <div className="flex flex-col gap-4">
+ <div
+ className={cn(
+ "flex flex-col gap-4",
+ hiddenFromReplay && HIDDEN_FROM_REPLAY,
+ )}
+ >
<div className="absolute right-3 top-3 z-10">
<OutputActions
items={items.map((item) => ({
diff --git a/autogpt_platform/frontend/src/lib/replay-privacy.ts b/autogpt_platform/frontend/src/lib/replay-privacy.ts
new file mode 100644
index 0000000000..8dc7cb6487
--- /dev/null
+++ b/autogpt_platform/frontend/src/lib/replay-privacy.ts
@@ -0,0 +1,11 @@
+/**
+ * Keeps an element and everything inside it out of session replays: PostHog
+ * blocks `ph-no-capture` (its default `blockClass`) and Sentry blocks
+ * `.sentry-block` (its default block selector). Blocking rather than masking,
+ * because the Text and Button atoms opt back in with `sentry-unmask`, which
+ * outranks a `sentry-mask` set further up the tree.
+ *
+ * Used where the screen shows what a connected account returned, such as
+ * Gmail or Drive content, which Google's Limited Use policy keeps from staff.
+ */
+export const HIDDEN_FROM_REPLAY = "ph-no-capture sentry-block";