Skip to content

Automate browser based workflows with AI

copilot: hard-block + safe-rewrite residual internal-vocab leaks on REPLY (SKY-10315) (#6538)

Andrew Neilson964859fb2d79
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 38d23a5f1..e33256d13 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -1013,7 +1013,7 @@ def _build_exit_result(
workflow_attempted=workflow_attempted,
unvalidated=False,
)
- output_policy_verdict = evaluate_output_policy(
+ raw_verdict = evaluate_output_policy(
request_policy=ctx.request_policy,
response_type="REPLY",
user_response=final_text,
@@ -1025,18 +1025,20 @@ def _build_exit_result(
unvalidated=False,
output_kind=output_kind,
)
- if not output_policy_verdict.allowed:
+ if not raw_verdict.allowed:
+ hard_block_verdict = hard_block_output_policy_verdict(raw_verdict)
+ soft_rewrite_reasons = [r for r in raw_verdict.reason_codes if r not in hard_block_verdict.reason_codes]
return _build_output_policy_blocked_result(
ctx,
- output_policy_verdict,
+ raw_verdict,
prior_global_llm_context=global_llm_context,
prior_workflow_yaml=verified_yaml,
output_policy_diagnostics=build_output_policy_diagnostics(
- raw_verdict=output_policy_verdict,
- final_verdict=output_policy_verdict,
- final_output_kind=_blocked_final_output_kind(output_policy_verdict),
- hard_block_reason_codes=list(output_policy_verdict.reason_codes),
- soft_rewrite_reason_codes=[],
+ raw_verdict=raw_verdict,
+ final_verdict=raw_verdict,
+ final_output_kind=_blocked_final_output_kind(raw_verdict),
+ hard_block_reason_codes=list(hard_block_verdict.reason_codes),
+ soft_rewrite_reason_codes=soft_rewrite_reasons,
),
)
return _finalize_result_with_blocker_override(
@@ -1096,7 +1098,7 @@ def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_context: s
workflow_attempted=True,
unvalidated=False,
)
- output_policy_verdict = evaluate_output_policy(
+ raw_verdict = evaluate_output_policy(
request_policy=ctx.request_policy,
response_type="REPLY",
user_response=final_text,
@@ -1108,18 +1110,20 @@ def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_context: s
unvalidated=False,
output_kind=output_kind,
)
- if not output_policy_verdict.allowed:
+ if not raw_verdict.allowed:
+ hard_block_verdict = hard_block_output_policy_verdict(raw_verdict)
+ soft_rewrite_reasons = [r for r in raw_verdict.reason_codes if r not in hard_block_verdict.reason_codes]
return _build_output_policy_blocked_result(
ctx,
- output_policy_verdict,
+ raw_verdict,
prior_global_llm_context=global_llm_context,
prior_workflow_yaml=verified_yaml,
output_policy_diagnostics=build_output_policy_diagnostics(
- raw_verdict=output_policy_verdict,
- final_verdict=output_policy_verdict,
- final_output_kind=_blocked_final_output_kind(output_policy_verdict),
- hard_block_reason_codes=list(output_policy_verdict.reason_codes),
- soft_rewrite_reason_codes=[],
+ raw_verdict=raw_verdict,
+ final_verdict=raw_verdict,
+ final_output_kind=_blocked_final_output_kind(raw_verdict),
+ hard_block_reason_codes=list(hard_block_verdict.reason_codes),
+ soft_rewrite_reason_codes=soft_rewrite_reasons,
),
)
return _finalize_result_with_blocker_override(
@@ -1236,6 +1240,10 @@ _INTERNAL_BLOCK_TAXONOMY_REPLY = (
"Describe the page action, data to collect, sign-in step, or check you want, and I'll translate that into "
"a supported workflow update."
)
+_INTERNAL_VOCAB_LEAK_REPLY = (
+ "Tell me what you'd like to do next — describe the page action, data to collect, sign-in step, "
+ "or check you want, and I'll translate that into a supported workflow update."
+)
_BLOCK_YAML_IN_REPLY_REWRITE_NO_PROPOSAL = (
"I drafted a change to the workflow but haven't applied it yet. Want me to update the workflow now?"
)
@@ -1919,23 +1927,30 @@ async def _translate_to_agent_result(
LOG.warning("Agent used inline REPLACE_WORKFLOW instead of update_workflow tool")
workflow_yaml = action_data.get("workflow_yaml", "")
if workflow_yaml:
- inline_policy_verdict = hard_block_output_policy_verdict(
- evaluate_output_policy(
- request_policy=ctx.request_policy,
- response_type=resp_type,
- user_response=str(user_response),
- workflow_yaml=workflow_yaml,
- tool_arguments=action_data,
- has_workflow_proposal=True,
- output_kind=CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL,
- )
+ inline_raw_verdict = evaluate_output_policy(
+ request_policy=ctx.request_policy,
+ response_type=resp_type,
+ user_response=str(user_response),
+ workflow_yaml=workflow_yaml,
+ tool_arguments=action_data,
+ has_workflow_proposal=True,
+ output_kind=CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL,
)
+ inline_policy_verdict = hard_block_output_policy_verdict(inline_raw_verdict)
if not inline_policy_verdict.allowed:
+ inline_diagnostics = build_output_policy_diagnostics(
+ raw_verdict=inline_raw_verdict,
+ final_verdict=inline_policy_verdict,
+ final_output_kind=_blocked_final_output_kind(inline_policy_verdict),
+ hard_block_reason_codes=list(inline_policy_verdict.reason_codes),
+ soft_rewrite_reason_codes=[],
+ )
return _build_output_policy_blocked_result(
ctx,
inline_policy_verdict,
prior_global_llm_context=global_llm_context,
prior_workflow_yaml=chat_request.workflow_yaml,
+ output_policy_diagnostics=inline_diagnostics,
)
# REPLACE_WORKFLOW bypasses the update_workflow tool guardrail, so
# policy and post-emission rejects run here before YAML processing.
@@ -2124,6 +2139,14 @@ async def _translate_to_agent_result(
user_response = _INTERNAL_BLOCK_TAXONOMY_REPLY
soft_rewrite_reasons.append(OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK)
output_policy_verdict.remove(OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK)
+ for _residual_vocab_reason in (
+ OutputPolicyReason.INTERNAL_CLASSIFIER_VOCAB_LEAK,
+ OutputPolicyReason.SELF_PRESCRIPTIVE_PHRASE_LEAK,
+ ):
+ if _residual_vocab_reason in output_policy_verdict.reason_codes:
+ user_response = _INTERNAL_VOCAB_LEAK_REPLY
+ soft_rewrite_reasons.append(_residual_vocab_reason)
+ output_policy_verdict.remove(_residual_vocab_reason)
if OutputPolicyReason.WORKFLOW_YAML_IN_REPLY in output_policy_verdict.reason_codes:
user_response = (
_BLOCK_YAML_IN_REPLY_REWRITE_WITH_PROPOSAL
@@ -2613,6 +2636,7 @@ def _output_policy_diagnostics_from_guardrail_exception(exc: BaseException) -> d
keys = {
"raw_output_kind",
"final_output_kind",
+ "raw_reason_codes",
"hard_block_reason_codes",
"soft_rewrite_reason_codes",
"raw_would_have_failed",
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py
index 9828740ee..44d374b21 100644
--- a/skyvern/forge/sdk/copilot/output_policy.py
+++ b/skyvern/forge/sdk/copilot/output_policy.py
@@ -105,6 +105,48 @@ _INFORMATIONAL_TAXONOMY_TERM_THRESHOLD = 3
_IDENTIFIER_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9_]*")
_IDENTIFIER_DELIMITERS = frozenset({"`", '"', "'"})
+_INTERNAL_CLASSIFIER_VOCAB_PHRASES = ("turnintent classified this turn as",)
+_INTERNAL_CLASSIFIER_REASON_CODE_LABEL_RE = re.compile(r"\bsafe_reason_code\s*[:=]")
+_INTERNAL_CLASSIFIER_REASON_CODE_PREFIXES = (
+ "turn_intent_",
+ "request_policy_",
+ "build_phase_",
+)
+_INTERNAL_CLASSIFIER_DELIMITED_NAMES = frozenset({"turnintent", "requestpolicy"})
+
+_INTERNAL_TOOL_PARAPHRASE_PHRASES = ("nudge turn", "nudge-turn")
+_INTERNAL_COPILOT_SENTINEL_PREFIX = "[copilot:"
+_LOOP_DETECTED_MARKER = "loop detected"
+
+_SELF_PRESCRIPTIVE_FIXED_PHRASE = "send me a normal instruction like"
+_SELF_PRESCRIPTIVE_IMPERATIVE_VERBS = frozenset({"send", "reply", "type", "respond"})
+_SELF_PRESCRIPTIVE_POLITE_PREFIXES = frozenset({"please", "kindly", "just", "now"})
+# Apostrophes inside contractions (`you'd`, `I'll`, `won't`) are not quote openers;
+# treat the single-quote characters as exemplar openers only at word boundaries.
+_SELF_PRESCRIPTIVE_UNCONDITIONAL_QUOTES = frozenset('"“”')
+_SELF_PRESCRIPTIVE_BOUNDARY_QUOTES = frozenset("'‘’")
+_SELF_PRESCRIPTIVE_QUOTED_WINDOW = 60
+_SELF_PRESCRIPTIVE_CONTINUATION_WINDOW = 120
+_SELF_PRESCRIPTIVE_POSITION_TERMINATORS = frozenset({"\n", ";", ".", "!", "?"})
+_SELF_PRESCRIPTIVE_POSITION_COORDINATORS = (", ", "and ", "or ")
+# Continuation cues distinguish chat-direction prescription ("type X to continue", "send X
+# next and I'll keep going") from imperative docs prose ("type X in the field", "send X
+# as JSON"). The fixed phrase above still fires without a cue; this guard only applies
+# to the looser verb+quoted-exemplar heuristic.
+_SELF_PRESCRIPTIVE_CONTINUATION_RE = re.compile(
+ r"\b(?:"
+ r"next|then|instead|keep going"
+ r"|to (?:continue|proceed|stop|abort|cancel|resume|retry)"
+ r"|and (?:i|we|you)['‘’]ll"
+ r")\b",
+ re.IGNORECASE,
+)
+
+# Response types whose `user_response` is rendered verbatim as the agent's final
+# message — REPLY and REPLACE_WORKFLOW. ASK_QUESTION is excluded because legitimate
+# clarifications ("Reply 'yes' to proceed") would false-positive the residual detectors.
+_USER_VISIBLE_REPLY_TYPES: frozenset[str] = frozenset({"REPLY", "REPLACE_WORKFLOW"})
+
@dataclass(frozen=True)
class ResponseScaffoldingNormalization:
@@ -134,6 +176,8 @@ class OutputPolicyReason(StrEnum):
INTERNAL_TOOL_INSTRUCTION_LEAK = "internal_tool_instruction_leak"
OUTPUT_POLICY_CONTEXT_MISSING = "output_policy_context_missing"
INTERNAL_BLOCK_TAXONOMY_LEAK = "internal_block_taxonomy_leak"
+ INTERNAL_CLASSIFIER_VOCAB_LEAK = "internal_classifier_vocab_leak"
+ SELF_PRESCRIPTIVE_PHRASE_LEAK = "self_prescriptive_phrase_leak"
WORKFLOW_YAML_IN_REPLY = "workflow_yaml_in_reply"
@@ -279,6 +323,7 @@ def build_output_policy_diagnostics(
return {
"raw_output_kind": raw_verdict.output_kind.value,
"final_output_kind": final_output_kind.value,
+ "raw_reason_codes": [reason.value for reason in raw_verdict.reason_codes],
"hard_block_reason_codes": [reason.value for reason in hard_block_reason_codes],
"soft_rewrite_reason_codes": [reason.value for reason in soft_rewrite_reason_codes],
"raw_would_have_failed": raw_would_have_failed,
@@ -357,8 +402,12 @@ def evaluate_output_policy(
and not _has_unvalidated_affordance(user_response)
):
verdict.add(OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE)
- if _contains_internal_block_taxonomy_leak(user_response, output_kind):
+ if _contains_internal_block_taxonomy_leak(user_response, output_kind, response_type):
verdict.add(OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK)
+ if response_type in _USER_VISIBLE_REPLY_TYPES and _contains_internal_classifier_vocab_leak(user_response):
+ verdict.add(OutputPolicyReason.INTERNAL_CLASSIFIER_VOCAB_LEAK)
+ if response_type in _USER_VISIBLE_REPLY_TYPES and _contains_self_prescriptive_phrase(user_response):
+ verdict.add(OutputPolicyReason.SELF_PRESCRIPTIVE_PHRASE_LEAK)
if response_type in ("REPLY", "ASK_QUESTION") and looks_like_workflow_yaml_in_chat(user_response):
verdict.add(OutputPolicyReason.WORKFLOW_YAML_IN_REPLY)
@@ -445,17 +494,29 @@ def _has_unvalidated_affordance(user_response: str | None) -> bool:
def _contains_internal_block_taxonomy_leak(
user_response: str | None,
output_kind: CopilotOutputKind,
+ response_type: str,
) -> bool:
if not user_response:
return False
if _contains_deprecated_block_identifier(user_response):
return True
+ if response_type in _USER_VISIBLE_REPLY_TYPES and _contains_internal_tool_vocab_leak(user_response):
+ return True
if output_kind != CopilotOutputKind.INFORMATIONAL_ANSWER:
return False
taxonomy_terms = _internal_block_taxonomy_terms(user_response)
return len(taxonomy_terms) >= _INFORMATIONAL_TAXONOMY_TERM_THRESHOLD
+def _contains_internal_tool_vocab_leak(user_response: str) -> bool:
+ lower = user_response.lower()
+ if _LOOP_DETECTED_MARKER in lower:
+ return True
+ if _INTERNAL_COPILOT_SENTINEL_PREFIX in lower:
+ return True
+ return any(phrase in lower for phrase in _INTERNAL_TOOL_PARAPHRASE_PHRASES)
+
+
def _contains_deprecated_block_identifier(text: str) -> bool:
tokens = [_compact_identifier_token(match.group(0)) for match in _IDENTIFIER_TOKEN_RE.finditer(text)]
if "taskv2" in tokens:
@@ -499,6 +560,110 @@ def _is_delimited_identifier(text: str, start: int, end: int) -> bool:
return left == text[end] and left in _IDENTIFIER_DELIMITERS
+def _contains_internal_classifier_vocab_leak(user_response: str | None) -> bool:
+ if not isinstance(user_response, str) or not user_response:
+ return False
+ lower = user_response.lower()
+ if any(phrase in lower for phrase in _INTERNAL_CLASSIFIER_VOCAB_PHRASES):
+ return True
+ if _INTERNAL_CLASSIFIER_REASON_CODE_LABEL_RE.search(lower):
+ return True
+ for match in _IDENTIFIER_TOKEN_RE.finditer(user_response):
+ token = match.group(0)
+ lowered = token.lower()
+ if "_" in token and any(lowered.startswith(prefix) for prefix in _INTERNAL_CLASSIFIER_REASON_CODE_PREFIXES):
+ return True
+ # CamelCase classifier names are internal vocabulary even without backticks;
+ # the lowercase forms (`turn intent`, `request policy`) remain natural prose.
+ if token in {"TurnIntent", "RequestPolicy"}:
+ return True
+ if lowered in _INTERNAL_CLASSIFIER_DELIMITED_NAMES and _is_delimited_identifier(
+ user_response, match.start(), match.end()
+ ):
+ return True
+ return False
+
+
+def _contains_self_prescriptive_phrase(user_response: str | None) -> bool:
+ if not isinstance(user_response, str) or not user_response:
+ return False
+ lower = user_response.lower()
+ if _SELF_PRESCRIPTIVE_FIXED_PHRASE in lower:
+ return True
+ for match in _IDENTIFIER_TOKEN_RE.finditer(user_response):
+ verb = match.group(0).lower()
+ if verb not in _SELF_PRESCRIPTIVE_IMPERATIVE_VERBS:
+ continue
+ if not _is_imperative_position(user_response, match.start()):
+ continue
+ window_end = min(len(user_response), match.end() + _SELF_PRESCRIPTIVE_QUOTED_WINDOW)
+ if not _contains_quoted_exemplar(user_response, match.end(), window_end):
+ continue
+ cue_end = min(len(user_response), match.end() + _SELF_PRESCRIPTIVE_CONTINUATION_WINDOW)
+ if _SELF_PRESCRIPTIVE_CONTINUATION_RE.search(user_response, match.end(), cue_end):
+ return True
+ return False
+
+
+def _contains_quoted_exemplar(text: str, window_start: int, window_end: int) -> bool:
+ for index in range(window_start, window_end):
+ char = text[index]
+ if char in _SELF_PRESCRIPTIVE_UNCONDITIONAL_QUOTES:
+ return True
+ if char in _SELF_PRESCRIPTIVE_BOUNDARY_QUOTES and not _is_embedded_apostrophe(text, index):
+ return True
+ return False
+
+
+def _is_embedded_apostrophe(text: str, index: int) -> bool:
+ if index == 0:
+ return False
+ return text[index - 1].isalnum()
+
+
+def _is_imperative_position(text: str, verb_start: int) -> bool:
+ return _is_position_at_sentence_start(text, verb_start) or _is_position_after_polite_prefix(text, verb_start)
+
+
+def _is_position_at_sentence_start(text: str, index: int) -> bool:
+ if index == 0:
+ return True
+ # Walk past intra-clause whitespace (spaces, tabs, leading newlines) so terminators
+ # followed by whitespace still count as sentence boundaries.
+ cursor = index - 1
+ while cursor >= 0 and text[cursor] in (" ", "\t"):
+ cursor -= 1
+ if cursor < 0:
+ return True
+ char = text[cursor]
+ if char in _SELF_PRESCRIPTIVE_POSITION_TERMINATORS:
+ return True
+ lower = text.lower()
+ for coordinator in _SELF_PRESCRIPTIVE_POSITION_COORDINATORS:
+ start = index - len(coordinator)
+ if start >= 0 and lower[start:index] == coordinator:
+ return True
+ return False
+
+
+def _is_position_after_polite_prefix(text: str, verb_start: int) -> bool:
+ if verb_start == 0:
+ return False
+ cursor = verb_start - 1
+ while cursor > 0 and text[cursor] == " ":
+ cursor -= 1
+ word_end = cursor + 1
+ word_start = word_end
+ while word_start > 0 and text[word_start - 1].isalpha():
+ word_start -= 1
+ if word_start == word_end:
+ return False
+ prefix = text[word_start:word_end].lower()
+ if prefix not in _SELF_PRESCRIPTIVE_POLITE_PREFIXES:
+ return False
+ return _is_position_at_sentence_start(text, word_start)
+
+
def _apply_credential_policy(
verdict: OutputPolicyVerdict,
request_policy: RequestPolicy,
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index f58e03aca..dfcb5d39c 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
CopilotOutputKind,
OutputPolicyReason,
OutputPolicyVerdict,
+ _contains_internal_tool_vocab_leak,
derive_output_kind,
evaluate_output_policy,
hard_block_output_policy_verdict,
@@ -707,6 +708,204 @@ def test_allows_single_task_block_reference_without_deprecated_identifier() -> N
assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK not in verdict.reason_codes
+ "user_response",
+ [
+ "TurnIntent classified this turn as `edit`, but we couldn't proceed. safe_reason_code=turn_intent_no_mutation_run_blocked.",
+ "Skipping: safe_reason_code=request_policy_clarification this turn.",
+ "Blocked by `TurnIntent`: please rephrase.",
+ "Hit turn_intent_unresolved_edit_target while routing.",
+ "RequestPolicy blocked this turn for clarification.",
+ "TurnIntent requires clarification before proceeding.",
+ "Trace shows safe_reason_code = turn_intent_no_mutation_run_blocked at exit.",
+ "Final safe_reason_code : request_policy_clarification reported.",
+ ],
+)
+def test_rejects_internal_classifier_vocab_leak(user_response: str) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=user_response,
+ )
+
+ assert OutputPolicyReason.INTERNAL_CLASSIFIER_VOCAB_LEAK in verdict.reason_codes
+
+
+ "user_response",
+ [
+ "The request policy resolved your saved credential automatically.",
+ "Use a turn-by-turn approach to walk through each step.",
+ "I'll classify the turn as edit if you confirm.",
+ "Need turnintent context? Tell me what you want to change.",
+ ],
+)
+def test_allows_benign_prose_referencing_classifier_terms(user_response: str) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=user_response,
+ )
+
+ assert OutputPolicyReason.INTERNAL_CLASSIFIER_VOCAB_LEAK not in verdict.reason_codes
+
+
+ "user_response",
+ [
+ "Saw [copilot:nudge] in the trace before the failure.",
+ "[copilot:screenshot] context truncated to keep this prompt small.",
+ "LOOP DETECTED: same tool dispatched three times in a row.",
+ "Loop Detected: same tool dispatched three times in a row.",
+ "loop detected: same tool dispatched three times in a row.",
+ "[Copilot:nudge] surfaced in the trace before the failure.",
+ "[COPILOT:NUDGE] surfaced in the trace before the failure.",
+ "Couldn't finish the diagnostic step on this nudge turn.",
+ ],
+)
+def test_rejects_internal_machinery_vocab_via_extended_taxonomy(user_response: str) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=user_response,
+ )
+
+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK in verdict.reason_codes
+
+
+ "user_response",
+ [
+ 'I\'ll call get_run_results(workflow_run_id="wr_123") and report back.',
+ "Next I need to run_blocks_and_collect_debug[block_labels=['login']].",
+ "Use `update_workflow` to apply that change.",
+ "Trace shows get_run_results was the last tool dispatched on this turn.",
+ "Falling back to list_credentials since the user did not specify one.",
+ ],
+)
+def test_rejects_internal_tool_name_leak_via_tool_instruction(user_response: str) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=user_response,
+ )
+
+ assert OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK in verdict.reason_codes
+ assert OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK in hard_block_output_policy_verdict(verdict).reason_codes
+
+
+ "user_response",
+ [
+ "I'll get the run results for you once it finishes.",
+ "Update the workflow to log in before extracting data.",
+ "List your saved credentials in the Credentials UI.",
+ "I'll take a screenshot once the page loads.",
+ "Nudge me if you want a different approach.",
+ ],
+)
+def test_allows_benign_prose_referencing_tool_names(user_response: str) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=user_response,
+ )
+
+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK not in verdict.reason_codes
+
+
+ "user_response",
+ [
+ "Use `update_workflow` to apply that change.",
+ "LOOP DETECTED: same tool dispatched three times in a row.",
+ "Couldn't finish the diagnostic step on this nudge turn.",
+ "Saw [copilot:nudge] in the trace before the failure.",
+ "Trace shows get_run_results was the last tool dispatched on this turn.",
+ ],
+)
+def test_extended_taxonomy_leak_skipped_on_ask_question(user_response: str) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="ASK_QUESTION",
+ user_response=user_response,
+ )
+
+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK not in verdict.reason_codes
+
+
+ "output_kind",
+ [
+ CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL,
+ CopilotOutputKind.WORKFLOW_UPDATE_PROPOSAL,
+ CopilotOutputKind.WORKFLOW_RUN_RESULT,
+ ],
+)
+def test_tool_name_leak_hard_blocks_across_output_kinds(
+ output_kind: CopilotOutputKind,
+) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response="Use `update_workflow` to apply that change.",
+ output_kind=output_kind,
+ )
+
+ assert OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK in verdict.reason_codes
+
+
+ "user_response",
+ [
+ "Send me a normal instruction like 'run it', and I'll continue.",
+ "Reply 'yes' to proceed.",
+ "Please send 'continue debugging' and I'll keep going.",
+ "Type 'cancel' to stop the workflow.",
+ "Kindly type 'cancel' to abort the current draft.",
+ " Reply 'yes' to proceed.",
+ "\tReply 'yes' to proceed.",
+ ],
+)
+def test_rejects_self_prescriptive_phrase(user_response: str) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=user_response,
+ )
+
+ assert OutputPolicyReason.SELF_PRESCRIPTIVE_PHRASE_LEAK in verdict.reason_codes
+
+
+ "user_response",
+ [
+ "I need the value for 'account_id' to continue.",
+ "The user can reply 'yes' to confirm.",
+ "I'll respond once I have 'account_id'.",
+ "I'll save the workflow once you confirm.",
+ "Please confirm whether to continue.",
+ "I have a draft workflow proposal. I need this before I can build and test it: which credential should I use?",
+ "Send me what you'd like changed and I'll update the workflow.",
+ "Reply once you've reviewed the draft.",
+ "Type the field name where you'd like the value populated.",
+ "Respond once you've decided which credential to use.",
+ 'Type "navigate" in the action field of the block.',
+ 'Send "data" as JSON to the endpoint.',
+ 'Reply objects include a status field like "success" or "error".',
+ "Reply 2's text is just a numeric ordinal, not a quote.",
+ ],
+)
+def test_allows_benign_prose_around_quoted_examples(user_response: str) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLY",
+ user_response=user_response,
+ )
+
+ assert OutputPolicyReason.SELF_PRESCRIPTIVE_PHRASE_LEAK not in verdict.reason_codes
+
+
def test_rejects_unapproved_credential_id_in_final_text() -> None:
verdict = evaluate_output_policy(
request_policy=_policy(),
@@ -1004,6 +1203,7 @@ def test_sdk_output_guardrail_hard_blocks_raw_secret_final_text() -> None:
assert diagnostics == {
"raw_output_kind": "informational_answer",
"final_output_kind": "refusal",
+ "raw_reason_codes": ["raw_secret_leak"],
"hard_block_reason_codes": ["raw_secret_leak"],
"soft_rewrite_reason_codes": [],
"raw_would_have_failed": True,
@@ -1719,6 +1919,7 @@ def test_translate_to_agent_result_rewrites_unbacked_workflow_claim() -> None:
assert agent_result.output_policy_diagnostics == {
"raw_output_kind": "informational_answer",
"final_output_kind": "clarification_request",
+ "raw_reason_codes": ["unbacked_workflow_delivery_claim", "missing_proposal_state"],
"hard_block_reason_codes": [],
"soft_rewrite_reason_codes": ["unbacked_workflow_delivery_claim", "missing_proposal_state"],
"raw_would_have_failed": True,
@@ -1767,6 +1968,7 @@ def test_translate_to_agent_result_rewrites_deprecated_block_taxonomy() -> None:
assert agent_result.output_policy_diagnostics == {
"raw_output_kind": "informational_answer",
"final_output_kind": "informational_answer",
+ "raw_reason_codes": ["internal_block_taxonomy_leak"],
"hard_block_reason_codes": [],
"soft_rewrite_reason_codes": ["internal_block_taxonomy_leak"],
"raw_would_have_failed": True,
@@ -1775,6 +1977,172 @@ def test_translate_to_agent_result_rewrites_deprecated_block_taxonomy() -> None:
}
+def test_translate_to_agent_result_rewrites_internal_classifier_vocab_leak() -> None:
+ leak = (
+ "TurnIntent classified this turn as `edit`, so the request couldn't continue. "
+ "safe_reason_code=turn_intent_no_mutation_run_blocked."
+ )
+ result = _fake_run_result({"type": "REPLY", "user_response": leak})
+
+ agent_result = asyncio.run(
+ agent_module._translate_to_agent_result(
+ result,
+ _ctx(),
+ global_llm_context=None,
+ chat_request=_chat_request(),
+ organization_id="org-1",
+ )
+ )
+
+ assert "TurnIntent" not in agent_result.user_response
+ assert "safe_reason_code" not in agent_result.user_response
+ assert "Tell me what you'd like to do next" in agent_result.user_response
+ diagnostics = agent_result.output_policy_diagnostics or {}
+ assert "internal_classifier_vocab_leak" in diagnostics["soft_rewrite_reason_codes"]
+ assert "internal_classifier_vocab_leak" in diagnostics["raw_reason_codes"]
+ assert diagnostics["raw_would_have_failed"] is True
+ assert diagnostics["contained_failure"] is True
+
+
+def test_translate_to_agent_result_hard_blocks_extended_taxonomy_tool_name_leak() -> None:
+ leak = "Trace shows `get_run_results` was the last tool dispatched on this turn."
+ result = _fake_run_result({"type": "REPLY", "user_response": leak})
+
+ agent_result = asyncio.run(
+ agent_module._translate_to_agent_result(
+ result,
+ _ctx(),
+ global_llm_context=None,
+ chat_request=_chat_request(),
+ organization_id="org-1",
+ )
+ )
+
+ assert "get_run_results" not in agent_result.user_response
+ assert "I could not safely return" in agent_result.user_response
+ diagnostics = agent_result.output_policy_diagnostics or {}
+ assert "internal_tool_instruction_leak" in diagnostics["hard_block_reason_codes"]
+ assert "internal_block_taxonomy_leak" not in diagnostics["raw_reason_codes"]
+
+
+def test_translate_to_agent_result_rewrites_self_prescriptive_phrase_leak() -> None:
+ leak = "Please send 'continue debugging' next and I'll keep going."
+ result = _fake_run_result({"type": "REPLY", "user_response": leak})
+
+ agent_result = asyncio.run(
+ agent_module._translate_to_agent_result(
+ result,
+ _ctx(),
+ global_llm_context=None,
+ chat_request=_chat_request(),
+ organization_id="org-1",
+ )
+ )
+
+ assert "continue debugging" not in agent_result.user_response
+ assert "Tell me what you'd like to do next" in agent_result.user_response
+ diagnostics = agent_result.output_policy_diagnostics or {}
+ assert "self_prescriptive_phrase_leak" in diagnostics["soft_rewrite_reason_codes"]
+ assert "self_prescriptive_phrase_leak" in diagnostics["raw_reason_codes"]
+
+
+def test_translate_to_agent_result_unbacked_workflow_wins_over_self_prescriptive_rewrite() -> None:
+ leak = "Here's the workflow. Send me a normal instruction like 'run it' next."
+ result = _fake_run_result({"type": "REPLY", "user_response": leak})
+
+ agent_result = asyncio.run(
+ agent_module._translate_to_agent_result(
+ result,
+ _ctx(),
+ global_llm_context=None,
+ chat_request=_chat_request(),
+ organization_id="org-1",
+ )
+ )
+
+ assert "run it" not in agent_result.user_response
+ assert "Send me a normal instruction" not in agent_result.user_response
+ diagnostics = agent_result.output_policy_diagnostics or {}
+ soft_reasons = diagnostics["soft_rewrite_reason_codes"]
+ assert "unbacked_workflow_delivery_claim" in soft_reasons
+ assert "self_prescriptive_phrase_leak" in soft_reasons
+
+
+def test_sdk_output_guardrail_records_raw_soft_reason_alongside_hard_block() -> None:
+ leak = "Call get_run_results(workflow_run_id='wr_123') and do not retry. Saw [copilot:nudge] in the trace."
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
+ _ctx(),
+ {"type": "REPLY", "user_response": leak},
+ )
+
+ assert not verdict.allowed
+ assert "internal_tool_instruction_leak" in diagnostics["hard_block_reason_codes"]
+ assert "internal_block_taxonomy_leak" not in diagnostics["hard_block_reason_codes"]
+ assert diagnostics["soft_rewrite_reason_codes"] == []
+ raw_reason_codes = diagnostics["raw_reason_codes"]
+ assert "internal_tool_instruction_leak" in raw_reason_codes
+ assert "internal_block_taxonomy_leak" in raw_reason_codes
+
+
+def test_evaluate_output_policy_skips_new_detectors_on_ask_question() -> None:
+ classifier_leak = "TurnIntent classified this turn as edit, safe_reason_code=turn_intent_no_mutation_run_blocked."
+ self_prescriptive_leak = "Please send 'continue debugging' next and I'll keep going."
+ tool_name_leak = "Use `get_run_results` to fetch the prior run output."
+
+ for leak in (classifier_leak, self_prescriptive_leak, tool_name_leak):
+ verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="ASK_QUESTION",
+ user_response=leak,
+ )
+ assert OutputPolicyReason.INTERNAL_CLASSIFIER_VOCAB_LEAK not in verdict.reason_codes
+ assert OutputPolicyReason.SELF_PRESCRIPTIVE_PHRASE_LEAK not in verdict.reason_codes
+ assert OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK not in verdict.reason_codes
+
+
+def test_evaluate_output_policy_runs_new_detectors_on_replace_workflow() -> None:
+ classifier_leak = "TurnIntent classified this turn as edit, safe_reason_code=turn_intent_no_mutation_run_blocked."
+ self_prescriptive_leak = "Please send 'continue debugging' next and I'll keep going."
+ tool_name_leak = "Use `get_run_results` to fetch the prior run output."
+
+ classifier_verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLACE_WORKFLOW",
+ user_response=classifier_leak,
+ )
+ assert OutputPolicyReason.INTERNAL_CLASSIFIER_VOCAB_LEAK in classifier_verdict.reason_codes
+
+ self_prescriptive_verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLACE_WORKFLOW",
+ user_response=self_prescriptive_leak,
+ )
+ assert OutputPolicyReason.SELF_PRESCRIPTIVE_PHRASE_LEAK in self_prescriptive_verdict.reason_codes
+
+ tool_name_verdict = evaluate_output_policy(
+ request_policy=_policy(),
+ response_type="REPLACE_WORKFLOW",
+ user_response=tool_name_leak,
+ )
+ assert OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK in tool_name_verdict.reason_codes
+
+
+ "user_response,expected",
+ [
+ ("LOOP DETECTED: same tool dispatched three times in a row.", True),
+ ("Saw [copilot:nudge] in the trace before the failure.", True),
+ ("Couldn't finish the diagnostic step on this nudge turn.", True),
+ ("nudge-turn was cleared before the next iteration.", True),
+ ("Update the workflow to log in before extracting data.", False),
+ ("I'll get the run results once it finishes.", False),
+ ("", False),
+ ],
+)
+def test_contains_internal_tool_vocab_leak_helper(user_response: str, expected: bool) -> None:
+ assert _contains_internal_tool_vocab_leak(user_response) is expected
+
+
def test_translate_to_agent_result_prioritizes_unbacked_workflow_claim_over_taxonomy_rewrite() -> None:
result = _fake_run_result(
{
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 373c6c4c5..776e6e477 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -984,6 +984,7 @@ async def test_unvalidated_timeout_wip_overrides_auto_accept(monkeypatch: pytest
output_policy_diagnostics={
"raw_output_kind": "informational_answer",
"final_output_kind": "informational_answer",
+ "raw_reason_codes": ["internal_block_taxonomy_leak"],
"hard_block_reason_codes": [],
"soft_rewrite_reason_codes": ["internal_block_taxonomy_leak"],
"raw_would_have_failed": True,