Skip to content

Automate browser based workflows with AI

Send Copilot tool schemas without generated titles on Claude and terra routes (SKY-18928) (#9065)

Andrew Neilsonb8b74484cfd2
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index f91a977da..56598ed85 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -1657,7 +1657,11 @@ _RAW_SECRET_CARD_NOT_CONNECTED = {
def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> str:
- has_keepable_draft = ctx.last_workflow is not None and bool(ctx.last_workflow_yaml)
+ has_keepable_draft = (
+ ctx.last_workflow is not None
+ and bool(ctx.last_workflow_yaml)
+ and not _is_saved_workflow(ctx, ctx.last_workflow)
+ )
keep_draft_affordance = " Keep the draft to iterate on, or discard." if has_keepable_draft else ""
block_count = ctx.last_update_block_count if isinstance(ctx.last_update_block_count, int) else None
positive_block_count = block_count if block_count is not None and block_count > 0 else None
@@ -1762,6 +1766,10 @@ def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> st
return user_response
+def _is_saved_workflow(ctx: CopilotContext, workflow: Workflow | None) -> bool:
+ return workflow is not None and any(workflow is saved for saved in ctx.saved_workflows_under_test)
+
+
def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:
"""Surface a proposal only when the current candidate passed its test run."""
run_status_clean = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is True
@@ -1770,6 +1778,7 @@ def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:
and ctx.last_workflow_yaml
and run_status_clean
and not artifact_health_blocked(ctx)
+ and not _is_saved_workflow(ctx, ctx.last_workflow)
):
return ctx.last_workflow, ctx.last_workflow_yaml
return None, None
@@ -2000,6 +2009,13 @@ def _make_agent_result(
]
}
)
+ if ctx is not None and _is_saved_workflow(ctx, kwargs.get("updated_workflow")):
+ LOG.warning(
+ "copilot dropped saved workflow offered as a proposal",
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ proposal_disposition=kwargs.get("proposal_disposition"),
+ )
+ kwargs.update(updated_workflow=None, workflow_yaml=None, proposal_disposition="no_proposal")
proposal_yaml = kwargs.get("workflow_yaml")
if isinstance(proposal_yaml, str):
proposal_yaml = derive_code_block_steps_in_yaml(proposal_yaml)
@@ -2950,7 +2966,12 @@ def _observed_facts_halt_reply(ctx: CopilotContext) -> str:
block_count = _workflow_block_count(ctx)
block_phrase = f"a {block_count}-block draft" if block_count else "a draft"
observed = _observed_page_sentence(ctx)
- if getattr(ctx, "last_workflow", None) is not None:
+ if _is_saved_workflow(ctx, ctx.last_workflow):
+ return (
+ f"I was testing the saved workflow when the turn ran out of time.{observed}"
+ " I haven't verified the results, so I'm not claiming them."
+ )
+ if ctx.last_workflow is not None:
return (
f"I built {block_phrase} and was still testing it when the turn ran out of time."
f"{observed} I haven't verified the results, so I'm not claiming them."
@@ -3063,13 +3084,19 @@ def _recorded_failure_reply(
# Only the run id proves a row existed: pre-run refusals can carry a synthetic
# overall_status="failed" even though execution never started.
run_created = bool(getattr(diagnosis_input, "workflow_run_id", None))
- if getattr(ctx, "last_workflow", None) is not None:
+ if ctx.last_workflow is not None:
+ saved = _is_saved_workflow(ctx, ctx.last_workflow)
+ tested = "I tested the saved workflow" if saved else f"I built {block_phrase} and tested it"
if (unrecoverable_stop and not run_created) or ctx.last_test_run_started is False:
+ if saved:
+ return f"I couldn't start a test run of the saved workflow: {reason}.{status_sentence}"
return f"I built {block_phrase}, but I couldn't start a test run: {reason}.{status_sentence}"
if test_attempted and test_failed and not unrecoverable_stop:
- return f"I built {block_phrase} and tested it, but the test failed: {reason}.{status_sentence}"
+ return f"{tested}, but the test failed: {reason}.{status_sentence}"
if test_attempted:
- return f"I built {block_phrase} and tested it, but the test couldn't finish: {reason}.{status_sentence}"
+ return f"{tested}, but the test couldn't finish: {reason}.{status_sentence}"
+ if saved:
+ return f"I couldn't verify the saved workflow: {reason}.{status_sentence}"
return f"I built {block_phrase}, but I couldn't verify it: {reason}.{status_sentence}"
return f"I couldn't finish the Copilot turn: {reason}.{status_sentence}"
@@ -3154,6 +3181,7 @@ def _build_wip_exit_result(
and ctx.last_good_workflow_yaml
and ctx.last_workflow is not ctx.last_good_workflow
and not ctx.last_test_suspicious_success
+ and not _is_saved_workflow(ctx, ctx.last_good_workflow)
):
append_failure_detail = recorded_failure_reply
held_reply = _tested_draft_reply(
@@ -3192,7 +3220,7 @@ def _build_wip_exit_result(
),
exit_site="wip_last_good_workflow",
)
- if ctx.last_workflow is not None and ctx.last_workflow_yaml:
+ if ctx.last_workflow is not None and ctx.last_workflow_yaml and not _is_saved_workflow(ctx, ctx.last_workflow):
full_test_ok = (
ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is True and not ctx.last_test_suspicious_success
)
@@ -3889,6 +3917,7 @@ async def _translate_to_agent_result(
and bool(ctx.last_failed_workflow_yaml or ctx.last_test_ok is False)
and not ctx.last_test_suspicious_success
and not blocker_active
+ and not _is_saved_workflow(ctx, ctx.last_good_workflow)
)
# ASK_QUESTION replies carry a specific clarifying question — often the
@@ -3925,7 +3954,13 @@ async def _translate_to_agent_result(
elif salvaged_reply:
last_workflow, last_workflow_yaml = ctx.last_good_workflow, ctx.last_good_workflow_yaml
unvalidated = True
- elif resp_type == "REPLY" and ctx.last_workflow is not None and ctx.last_workflow_yaml and not blocker_active:
+ elif (
+ resp_type == "REPLY"
+ and ctx.last_workflow is not None
+ and ctx.last_workflow_yaml
+ and not blocker_active
+ and not _is_saved_workflow(ctx, ctx.last_workflow)
+ ):
# Failures are often environmental (captcha, transient block); surface the draft so the user can keep iterating.
last_workflow = ctx.last_workflow
last_workflow_yaml = ctx.last_workflow_yaml
@@ -4033,7 +4068,9 @@ async def _translate_to_agent_result(
staged_workflow=ctx.staged_workflow,
canonical_was_persisted_due_to_param_change=ctx.canonical_was_persisted_due_to_param_change,
total_tokens=ctx.total_tokens_used,
- clear_proposed_workflow=(resp_type == "ASK_QUESTION" and last_workflow is None),
+ clear_proposed_workflow=(
+ resp_type == "ASK_QUESTION" and last_workflow is None and not _is_saved_workflow(ctx, ctx.last_workflow)
+ ),
proposal_disposition=(
"review_tested"
if chat_request.product_action == "test_end_to_end"
@@ -4539,7 +4576,12 @@ def _agent_output_to_text(agent_output: Any) -> str:
def _should_surface_untested_draft_despite_question(ctx: CopilotContext, response_type: str) -> bool:
- if response_type != "ASK_QUESTION" or ctx.last_workflow is None or not ctx.last_workflow_yaml:
+ if (
+ response_type != "ASK_QUESTION"
+ or ctx.last_workflow is None
+ or not ctx.last_workflow_yaml
+ or _is_saved_workflow(ctx, ctx.last_workflow)
+ ):
return False
return ctx.last_test_ok is None and ctx.last_run_skipped_unbound_credentials
@@ -4581,16 +4623,17 @@ def _evaluate_copilot_final_output_policy(
) -> tuple[OutputPolicyVerdict, str, dict[str, Any]]:
action_data, response_type, policy_user_response = _final_output_policy_reply(agent_output)
+ candidate_workflow = None if _is_saved_workflow(ctx, ctx.last_workflow) else ctx.last_workflow
workflow_yaml = None
if response_type == "REPLACE_WORKFLOW" and isinstance(action_data.get("workflow_yaml"), str):
workflow_yaml = action_data["workflow_yaml"]
- elif isinstance(getattr(ctx, "last_workflow_yaml", None), str):
+ elif candidate_workflow is not None and isinstance(ctx.last_workflow_yaml, str):
workflow_yaml = ctx.last_workflow_yaml
workflow_attempted = ctx.has_genuine_workflow_attempt()
_log_output_policy_parity(
ctx,
- has_workflow_proposal=bool(workflow_yaml or ctx.last_workflow is not None),
+ has_workflow_proposal=bool(workflow_yaml or candidate_workflow is not None),
workflow_attempted=workflow_attempted,
)
surface_untested_draft = _should_surface_untested_draft_despite_question(ctx, response_type)
@@ -4598,7 +4641,7 @@ def _evaluate_copilot_final_output_policy(
if surface_untested_draft:
policy_user_response = _rewrite_failed_test_response(policy_user_response, ctx)
updated_workflow_for_kind = (
- ctx.last_workflow if ctx.last_workflow is not None else WORKFLOW_PRESENT_SENTINEL if workflow_yaml else None
+ candidate_workflow if candidate_workflow is not None else WORKFLOW_PRESENT_SENTINEL if workflow_yaml else None
)
if surface_untested_draft:
output_kind = (
@@ -4621,7 +4664,7 @@ def _evaluate_copilot_final_output_policy(
user_response=policy_user_response,
global_llm_context=action_data.get("global_llm_context"),
workflow_yaml=workflow_yaml,
- has_workflow_proposal=bool(workflow_yaml or ctx.last_workflow is not None),
+ has_workflow_proposal=bool(workflow_yaml or candidate_workflow is not None),
workflow_was_persisted=ctx.workflow_persisted,
workflow_attempted=workflow_attempted,
output_kind=output_kind,
@@ -4790,7 +4833,12 @@ def _build_output_policy_blocked_result(
local_blocker_signal = ctx.blocker_signal if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal) else None
blocker_active = local_blocker_signal is not None and local_blocker_signal.renders_final_reply
preserved_workflow = (
- ctx.last_workflow if ctx.last_workflow is not None and ctx.last_workflow_yaml and not blocker_active else None
+ ctx.last_workflow
+ if ctx.last_workflow is not None
+ and ctx.last_workflow_yaml
+ and not blocker_active
+ and not _is_saved_workflow(ctx, ctx.last_workflow)
+ else None
)
preserved_workflow_yaml = ctx.last_workflow_yaml if preserved_workflow is not None else None
output_policy_reasons = list(verdict.reason_codes if evaluated_reason_codes is None else evaluated_reason_codes)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index c846f6c94..3ae0c66e8 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -1368,6 +1368,8 @@ class CopilotContext(AgentContext):
private_workflow_settings: dict[str, Any] = field(default_factory=dict, repr=False)
last_workflow: Workflow | None = None
last_workflow_yaml: str | None = None
+ # Copies of the saved workflow a full test loaded this turn; matched by identity, never offered as a proposal.
+ saved_workflows_under_test: list[Workflow] = field(default_factory=list)
# Always False under staging; ``has_staged_proposal`` carries the signal.
workflow_persisted: bool = False
last_update_block_count: int | None = None
diff --git a/skyvern/forge/sdk/copilot/model_telemetry.py b/skyvern/forge/sdk/copilot/model_telemetry.py
index ad18d0915..3e2bcdab2 100644
--- a/skyvern/forge/sdk/copilot/model_telemetry.py
+++ b/skyvern/forge/sdk/copilot/model_telemetry.py
@@ -10,7 +10,7 @@ import time
from collections.abc import AsyncIterator, Callable, Iterator
from dataclasses import dataclass, field, replace
from types import SimpleNamespace
-from typing import Any, Literal, cast, overload
+from typing import Any, Literal, TypeVar, cast, overload
from urllib.parse import urlparse
import litellm
@@ -25,7 +25,7 @@ from agents.models.chatcmpl_converter import Converter
from agents.models.fake_id import FAKE_RESPONSES_ID
from agents.models.interface import ModelTracing
from agents.models.openai_responses import Converter as OpenAIResponsesConverter
-from agents.tool import Tool
+from agents.tool import FunctionTool, Tool
from agents.tracing.span_data import GenerationSpanData
from agents.tracing.spans import Span
from litellm.completion_extras import responses_api_bridge
@@ -49,6 +49,7 @@ from skyvern.forge.sdk.api.llm.copilot_model_usage import (
normalize_gen_ai_provider,
)
from skyvern.forge.sdk.api.llm.litellm_pricing import long_prompt_cost_multiplier
+from skyvern.forge.sdk.api.llm.utils import is_anthropic_model
from skyvern.forge.sdk.copilot.cache_envelope import (
CacheableSystemInstructions,
ExplicitCacheEnvelope,
@@ -611,6 +612,40 @@ def _escape_ref_tool_outputs_for_gemini(
return rewritten
+_SCHEMA_LITERALS = frozenset({"default", "const", "enum", "examples"})
+
+
+def _drop_schema_titles(node: Any) -> Any:
+ if isinstance(node, list):
+ return [_drop_schema_titles(item) for item in node]
+ if not isinstance(node, dict):
+ return node
+ return {
+ key: value if key in _SCHEMA_LITERALS else _drop_schema_titles(value)
+ for key, value in node.items()
+ if not (key == "title" and isinstance(value, str))
+ }
+
+
+_ToolT = TypeVar("_ToolT", bound=Tool)
+
+
+def drop_tool_schema_titles(tools: list[_ToolT], model: str, model_settings: ModelSettings) -> list[_ToolT]:
+ # Pydantic's generated titles carry nothing the property names don't. Offline replay showed no behavior change
+ # without them on Claude and terra; gpt-6.1-sol replied less often after a passing test run, so other models keep them.
+ if not all(
+ is_anthropic_model(candidate) or "gpt-5.6-terra" in candidate
+ for candidate in model_chain(model, model_settings)
+ ):
+ return tools
+ return [
+ replace(tool, params_json_schema=_drop_schema_titles(tool.params_json_schema))
+ if isinstance(tool, FunctionTool)
+ else tool
+ for tool in tools
+ ]
+
+
class CopilotLitellmModel(LitellmModel):
def __init__(
self,
@@ -636,7 +671,6 @@ class CopilotLitellmModel(LitellmModel):
conversation_id: str | None = None,
prompt: Any | None = None,
) -> ModelResponse:
- attach_tool_surface_to_pending_capture(tools)
with _model_call_telemetry_scope(
self.next_model_call_index(),
model=self.model,
@@ -668,7 +702,6 @@ class CopilotLitellmModel(LitellmModel):
conversation_id: str | None = None,
prompt: Any | None = None,
) -> AsyncIterator[TResponseStreamEvent]:
- attach_tool_surface_to_pending_capture(tools)
with _model_call_telemetry_scope(
self.next_model_call_index(),
model=self.model,
@@ -732,6 +765,8 @@ class CopilotLitellmModel(LitellmModel):
prompt: Any | None = None,
) -> LiteLLMModelResponse | tuple[Response, AsyncStream[ChatCompletionChunk]]:
input = _escape_ref_tool_outputs_for_gemini(input, self.model, model_settings)
+ tools = drop_tool_schema_titles(tools, self.model, model_settings)
+ attach_tool_surface_to_pending_capture(tools)
anchors = stable_prefix_anchors(input) if isinstance(input, list) else []
anchor = anchors[-1] if anchors else None
explicit_cache_envelope = build_explicit_cache_envelope(
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 422e211ba..c8ce9c30c 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -22,7 +22,6 @@ from typing import Any, Literal, NamedTuple, NotRequired, TypedDict
from urllib.parse import urlparse
import structlog
-import yaml
from fastapi import HTTPException
from skyvern.constants import SCRUBBED_VALUE
@@ -240,6 +239,7 @@ from skyvern.forge.sdk.copilot.workflow_yaml import (
_process_workflow_yaml,
runner_code_block_associations,
stored_block_code,
+ workflow_to_copilot_yaml,
)
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.db.enums import BrowserSeedSource
@@ -4043,8 +4043,7 @@ async def _run_blocks_and_collect_debug(
workflow = snapshot.workflow
execution = _RunExecution(
snapshot=snapshot,
- workflow_yaml=snapshot.workflow_yaml
- or yaml.safe_dump({"workflow_definition": workflow.workflow_definition.model_dump(mode="json")}),
+ workflow_yaml=snapshot.workflow_yaml or workflow_to_copilot_yaml(workflow),
metadata=copy.deepcopy(ctx.code_artifact_metadata),
associations=copy.deepcopy(ctx.runner_code_block_associations_by_label),
source_at_start=(
@@ -4081,6 +4080,7 @@ async def _run_blocks_and_collect_debug(
# Existing verification consumers need the same source this full test selected.
ctx.last_workflow = snapshot.workflow
ctx.last_workflow_yaml = execution.workflow_yaml
+ ctx.saved_workflows_under_test.append(snapshot.workflow)
for label in block_labels:
if not workflow.get_output_parameter(label):
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index e1e25e23f..753276517 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -5961,6 +5961,7 @@ class _CredentialWorkflow(BaseModel):
"""Raw definition fixture for credential admission, with snapshot copy semantics."""
workflow_id: str = "wf-1"
+ workflow_permanent_id: str = "wfp-1"
workflow_definition: _CredentialWorkflowDefinition
output_labels: set[str]
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index fdab6c418..cafe39c76 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -30,6 +30,7 @@ from skyvern.forge.sdk.copilot.agent import (
_prior_run_debug_text,
_recorded_build_test_outcome_prompt,
_review_projection_for,
+ _translate_to_agent_result,
)
from skyvern.forge.sdk.copilot.build_test_connect_failure import build_test_connect_failure_sentence
from skyvern.forge.sdk.copilot.build_test_outcome import (
@@ -67,9 +68,10 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
)
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
-from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
+from skyvern.forge.sdk.copilot.context import AgentResult, CodeAuthoringRepairContext, CopilotContext
from skyvern.forge.sdk.copilot.enforcement import _summarize_tool_output
from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
+from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind
from skyvern.forge.sdk.copilot.output_utils import (
_BUILD_TEST_PACKET_MAX_CHARS,
_INTERNAL_RUN_OUTCOME_RECORDED_KEY,
@@ -82,6 +84,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
project_direct_test_handoff_packet_for_llm,
sanitize_tool_result_for_llm,
)
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprints
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
@@ -8923,3 +8926,154 @@ async def test_an_unprovable_executed_version_is_unavailable_and_never_falls_bac
"workflow_id": "wf-1",
"reason": reason,
}
+
+
+_SAVED_SIGN_IN_AND_SEARCH_YAML = """
+title: sign in and search
+workflow_definition:
+ version: 2
+ parameters:
+ - key: login_credentials
+ parameter_type: workflow
+ workflow_parameter_type: credential_id
+ default_value: cred_saved
+ - key: search_term
+ parameter_type: workflow
+ workflow_parameter_type: string
+ default_value: widgets
+ blocks:
+ - block_type: login
+ label: sign_in
+ url: https://example.com/login
+ parameter_keys: [login_credentials]
+ next_block_label: search
+ - block_type: code
+ label: search
+ parameter_keys: [login_credentials, search_term]
+ code: |
+ await page.fill("#q", search_term)
+"""
+
+_PROPOSAL_OUTPUT_KINDS = {
+ CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL.value,
+ CopilotOutputKind.WORKFLOW_UPDATE_PROPOSAL.value,
+}
+
+
+def _saved_sign_in_ctx() -> CopilotContext:
+ return make_copilot_ctx(
+ browser_session_id="pbs_chat",
+ workflow_copilot_chat_id="chat-1",
+ request_policy=RequestPolicy(persisted_workflow_credential_ids=["cred_saved"]),
+ )
+
+
+async def _blank_browser_retest_of_saved_workflow(
+ monkeypatch: pytest.MonkeyPatch, ctx: CopilotContext, status: str
+) -> None:
+ await install_run_blocks_harness(
+ monkeypatch,
+ workflow_yaml=_SAVED_SIGN_IN_AND_SEARCH_YAML,
+ polled_status=status,
+ dispatch_to_worker=True,
+ terminal_blocks=[
+ terminal_extraction_block("completed", label="sign_in"),
+ terminal_extraction_block(status, label="search"),
+ ],
+ )
+ forge_app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final = AsyncMock()
+ monkeypatch.setattr(run_execution_module, "_credential_ids_validation_error", AsyncMock(return_value=None))
+
+ async def mint_blank_browser(acquisition_ctx: CopilotContext, *, fresh: bool, **_kwargs: object) -> None:
+ acquisition_ctx.browser_session_id = "pbs_run"
+
+ monkeypatch.setattr(run_execution_module, "acquire_build_test_browser_session", mint_blank_browser)
+ workflow_service_module.prepare_workflow.return_value.browser_session_id = "pbs_run"
+ result = await run_execution_module.run_workflow_end_to_end(ctx)
+ assert result["data"]["execution_source"]["source_kind"] == "canonical", result
+
+
+async def test_canonical_blank_browser_retest_records_yaml_the_accept_resolver_keeps_bindings(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _saved_sign_in_ctx()
+
+ await _blank_browser_retest_of_saved_workflow(monkeypatch, ctx, "failed")
+
+ assert ctx.last_workflow is not None and ctx.last_workflow_yaml
+ proposal = workflow_copilot_route._build_proposed_workflow_data(
+ ctx.last_workflow,
+ AgentResult(
+ user_response="",
+ updated_workflow=ctx.last_workflow,
+ workflow_yaml=ctx.last_workflow_yaml,
+ global_llm_context=None,
+ ),
+ )
+ resolved = await workflow_copilot_route._resolve_proposal_for_accept(
+ proposal, organization_id="org-1", workflow_permanent_id="wfp-1"
+ )
+
+ blocks = {block.label: block for block in resolved.workflow_definition.blocks}
+ assert blocks["search"].parameter_keys == ["login_credentials", "search_term"]
+ assert blocks["sign_in"].parameter_keys == ["login_credentials"]
+ assert {parameter.key for parameter in resolved.workflow_definition.parameters} >= {
+ "login_credentials",
+ "search_term",
+ }
+ assert ctx.latest_diagnosis_repair_contract is not None
+ assert {
+ lever.credential_id
+ for lever in ctx.latest_diagnosis_repair_contract.levers
+ if lever.mechanism == "credential_update"
+ } == {"cred_saved"}
+
+
[email protected]("run_statuses", [("completed",), ("failed",), ("completed", "failed")])
+ ("response_type", "product_action"),
+ [("REPLY", None), ("ASK_QUESTION", None), ("REPLY", "test_end_to_end")],
+)
+async def test_canonical_blank_browser_retests_stage_no_proposal_on_any_exit(
+ monkeypatch: pytest.MonkeyPatch, run_statuses: tuple[str, ...], response_type: str, product_action: str | None
+) -> None:
+ ctx = _saved_sign_in_ctx()
+
+ for status in run_statuses:
+ await _blank_browser_retest_of_saved_workflow(monkeypatch, ctx, status)
+
+ failed = run_statuses[-1] == "failed"
+ assert ctx.last_workflow is not None and ctx.last_test_ok is not failed
+ chat_request = SimpleNamespace(
+ workflow_id="wf-1",
+ workflow_permanent_id="wfp-1",
+ workflow_copilot_chat_id="chat-1",
+ workflow_run_id=None,
+ workflow_yaml="",
+ product_action=product_action,
+ )
+ normal_exit = await _translate_to_agent_result(
+ SimpleNamespace(
+ final_output=json.dumps({"type": response_type, "user_response": "I re-ran the saved workflow."}),
+ new_items=[],
+ ),
+ ctx,
+ None,
+ chat_request,
+ "org-1",
+ )
+ wip_exit = _build_wip_exit_result(ctx, None, default_reply="Stopped.")
+
+ for result in (normal_exit, wip_exit):
+ assert (result.updated_workflow, result.workflow_yaml, result.staged_workflow) == (None, None, None)
+ assert result.proposal_disposition == "no_proposal"
+ assert result.clear_proposed_workflow is False
+ assert (result.output_policy_diagnostics or {}).get("final_output_kind") not in _PROPOSAL_OUTPUT_KINDS
+ assert ctx.last_workflow is not None
+ if failed:
+ ctx.saved_workflows_under_test.clear()
+ draft_wip_exit = _build_wip_exit_result(ctx, None, default_reply="Stopped.")
+ assert draft_wip_exit.updated_workflow is ctx.last_workflow
+ assert wip_exit.user_response != draft_wip_exit.user_response
diff --git a/tests/unit/test_copilot_model_telemetry.py b/tests/unit/test_copilot_model_telemetry.py
index adcbbd2dc..789f65cac 100644
--- a/tests/unit/test_copilot_model_telemetry.py
+++ b/tests/unit/test_copilot_model_telemetry.py
@@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Iterator
from dataclasses import dataclass
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from types import SimpleNamespace
-from typing import Any, Self, cast
+from typing import Any, Literal, Self, cast
from unittest.mock import AsyncMock, MagicMock
import httpx
@@ -226,15 +226,13 @@ async def test_model_adapter_attaches_exact_tools_for_nonstream_and_stream_reque
) -> None:
attached: list[list[Any]] = []
- async def fake_get_response(*_args: Any, **_kwargs: Any) -> SimpleNamespace:
- return SimpleNamespace(output=[])
-
- async def fake_stream_response(*_args: Any, **_kwargs: Any) -> AsyncIterator[str]:
- yield "done"
+ async def fake_acompletion(**kwargs: Any) -> Any:
+ if kwargs.get("stream"):
+ return cast(AsyncStream[ChatCompletionChunk], _ChunkStream(_stream_chunks()))
+ return _completion()
monkeypatch.setattr(model_telemetry_module, "attach_tool_surface_to_pending_capture", attached.append)
- monkeypatch.setattr(LitellmModel, "get_response", fake_get_response)
- monkeypatch.setattr(LitellmModel, "stream_response", fake_stream_response)
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
model = CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1)
await model.get_response(
@@ -260,7 +258,7 @@ async def test_model_adapter_attaches_exact_tools_for_nonstream_and_stream_reque
]
assert attached == [[_lookup_number], [_lookup_number]]
- assert streamed == ["done"]
+ assert streamed
@pytest.mark.asyncio
@@ -280,6 +278,62 @@ async def test_nonstream_capture_preserves_request_and_result(monkeypatch: pytes
assert current_model_call_telemetry() is None
+@function_tool(strict_mode=False)
+def _pick(title: str | None = None, mode: Literal["a", "b"] | None = None) -> str:
+ return f"{title}{mode}"
+
+
+ ("model_name", "drops_titles"),
+ [("anthropic/claude-sonnet-5-5", True), ("azure/gpt-5.6-terra", True), ("gpt-6.1-sol", False)],
+)
+async def test_sent_tool_schemas_drop_generated_titles_only_on_replay_checked_models(
+ monkeypatch: pytest.MonkeyPatch,
+ model_name: str,
+ drops_titles: bool,
+) -> None:
+ sent: list[dict[str, Any]] = []
+ attached: list[list[Any]] = []
+
+ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:
+ sent.append(kwargs["tools"][0]["function"]["parameters"])
+ return _completion()
+
+ async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse:
+ sent.append(kwargs["tools"][0]["parameters"])
+ return _responses_completion()
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+ monkeypatch.setattr("litellm.aresponses", fake_aresponses)
+ monkeypatch.setattr(model_telemetry_module, "attach_tool_surface_to_pending_capture", attached.append)
+ original = copy.deepcopy(_pick.params_json_schema)
+
+ await CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1).get_response(
+ system_instructions=_CACHED_PROMPT,
+ input=[{"role": "user", "content": "Pick one"}],
+ model_settings=ModelSettings(temperature=0, include_usage=True, extra_args=_GPT_REASONING),
+ tools=[_pick],
+ output_schema=None,
+ handoffs=[],
+ tracing=ModelTracing.DISABLED,
+ )
+
+ assert _pick.params_json_schema == original
+ assert sent == [
+ {
+ "properties": {
+ "title": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": None},
+ "mode": {"anyOf": [{"enum": ["a", "b"], "type": "string"}, {"type": "null"}], "default": None},
+ },
+ "type": "object",
+ }
+ if drops_titles
+ else original
+ ]
+ assert [[tool.params_json_schema for tool in tools] for tools in attached] == [sent]
+
+
@pytest.mark.asyncio
async def test_direct_gpt56_adds_one_stable_prefix_breakpoint_without_changing_prompt_text(
monkeypatch: pytest.MonkeyPatch,