Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Send Copilot tool schemas without generated titles on Claude and terra routes (SKY-18928) (#9065)
Andrew Neilsonb8b74484cfd2
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex f91a977da..56598ed85 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -1657,7 +1657,11 @@ _RAW_SECRET_CARD_NOT_CONNECTED = {def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> str:- has_keepable_draft = ctx.last_workflow is not None and bool(ctx.last_workflow_yaml)+ has_keepable_draft = (+ ctx.last_workflow is not None+ and bool(ctx.last_workflow_yaml)+ and not _is_saved_workflow(ctx, ctx.last_workflow)+ )keep_draft_affordance = " Keep the draft to iterate on, or discard." if has_keepable_draft else ""block_count = ctx.last_update_block_count if isinstance(ctx.last_update_block_count, int) else Nonepositive_block_count = block_count if block_count is not None and block_count > 0 else None@@ -1762,6 +1766,10 @@ def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> streturn user_response+def _is_saved_workflow(ctx: CopilotContext, workflow: Workflow | None) -> bool:+ return workflow is not None and any(workflow is saved for saved in ctx.saved_workflows_under_test)++def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:"""Surface a proposal only when the current candidate passed its test run."""run_status_clean = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is True@@ -1770,6 +1778,7 @@ def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:and ctx.last_workflow_yamland run_status_cleanand not artifact_health_blocked(ctx)+ and not _is_saved_workflow(ctx, ctx.last_workflow)):return ctx.last_workflow, ctx.last_workflow_yamlreturn None, None@@ -2000,6 +2009,13 @@ def _make_agent_result(]})+ if ctx is not None and _is_saved_workflow(ctx, kwargs.get("updated_workflow")):+ LOG.warning(+ "copilot dropped saved workflow offered as a proposal",+ workflow_permanent_id=ctx.workflow_permanent_id,+ proposal_disposition=kwargs.get("proposal_disposition"),+ )+ kwargs.update(updated_workflow=None, workflow_yaml=None, proposal_disposition="no_proposal")proposal_yaml = kwargs.get("workflow_yaml")if isinstance(proposal_yaml, str):proposal_yaml = derive_code_block_steps_in_yaml(proposal_yaml)@@ -2950,7 +2966,12 @@ def _observed_facts_halt_reply(ctx: CopilotContext) -> str:block_count = _workflow_block_count(ctx)block_phrase = f"a {block_count}-block draft" if block_count else "a draft"observed = _observed_page_sentence(ctx)- if getattr(ctx, "last_workflow", None) is not None:+ if _is_saved_workflow(ctx, ctx.last_workflow):+ return (+ f"I was testing the saved workflow when the turn ran out of time.{observed}"+ " I haven't verified the results, so I'm not claiming them."+ )+ if ctx.last_workflow is not None:return (f"I built {block_phrase} and was still testing it when the turn ran out of time."f"{observed} I haven't verified the results, so I'm not claiming them."@@ -3063,13 +3084,19 @@ def _recorded_failure_reply(# Only the run id proves a row existed: pre-run refusals can carry a synthetic# overall_status="failed" even though execution never started.run_created = bool(getattr(diagnosis_input, "workflow_run_id", None))- if getattr(ctx, "last_workflow", None) is not None:+ if ctx.last_workflow is not None:+ saved = _is_saved_workflow(ctx, ctx.last_workflow)+ tested = "I tested the saved workflow" if saved else f"I built {block_phrase} and tested it"if (unrecoverable_stop and not run_created) or ctx.last_test_run_started is False:+ if saved:+ return f"I couldn't start a test run of the saved workflow: {reason}.{status_sentence}"return f"I built {block_phrase}, but I couldn't start a test run: {reason}.{status_sentence}"if test_attempted and test_failed and not unrecoverable_stop:- return f"I built {block_phrase} and tested it, but the test failed: {reason}.{status_sentence}"+ return f"{tested}, but the test failed: {reason}.{status_sentence}"if test_attempted:- return f"I built {block_phrase} and tested it, but the test couldn't finish: {reason}.{status_sentence}"+ return f"{tested}, but the test couldn't finish: {reason}.{status_sentence}"+ if saved:+ return f"I couldn't verify the saved workflow: {reason}.{status_sentence}"return f"I built {block_phrase}, but I couldn't verify it: {reason}.{status_sentence}"return f"I couldn't finish the Copilot turn: {reason}.{status_sentence}"@@ -3154,6 +3181,7 @@ def _build_wip_exit_result(and ctx.last_good_workflow_yamland ctx.last_workflow is not ctx.last_good_workflowand not ctx.last_test_suspicious_success+ and not _is_saved_workflow(ctx, ctx.last_good_workflow)):append_failure_detail = recorded_failure_replyheld_reply = _tested_draft_reply(@@ -3192,7 +3220,7 @@ def _build_wip_exit_result(),exit_site="wip_last_good_workflow",)- if ctx.last_workflow is not None and ctx.last_workflow_yaml:+ if ctx.last_workflow is not None and ctx.last_workflow_yaml and not _is_saved_workflow(ctx, ctx.last_workflow):full_test_ok = (ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is True and not ctx.last_test_suspicious_success)@@ -3889,6 +3917,7 @@ async def _translate_to_agent_result(and bool(ctx.last_failed_workflow_yaml or ctx.last_test_ok is False)and not ctx.last_test_suspicious_successand not blocker_active+ and not _is_saved_workflow(ctx, ctx.last_good_workflow))# ASK_QUESTION replies carry a specific clarifying question — often the@@ -3925,7 +3954,13 @@ async def _translate_to_agent_result(elif salvaged_reply:last_workflow, last_workflow_yaml = ctx.last_good_workflow, ctx.last_good_workflow_yamlunvalidated = True- elif resp_type == "REPLY" and ctx.last_workflow is not None and ctx.last_workflow_yaml and not blocker_active:+ elif (+ resp_type == "REPLY"+ and ctx.last_workflow is not None+ and ctx.last_workflow_yaml+ and not blocker_active+ and not _is_saved_workflow(ctx, ctx.last_workflow)+ ):# Failures are often environmental (captcha, transient block); surface the draft so the user can keep iterating.last_workflow = ctx.last_workflowlast_workflow_yaml = ctx.last_workflow_yaml@@ -4033,7 +4068,9 @@ async def _translate_to_agent_result(staged_workflow=ctx.staged_workflow,canonical_was_persisted_due_to_param_change=ctx.canonical_was_persisted_due_to_param_change,total_tokens=ctx.total_tokens_used,- clear_proposed_workflow=(resp_type == "ASK_QUESTION" and last_workflow is None),+ clear_proposed_workflow=(+ resp_type == "ASK_QUESTION" and last_workflow is None and not _is_saved_workflow(ctx, ctx.last_workflow)+ ),proposal_disposition=("review_tested"if chat_request.product_action == "test_end_to_end"@@ -4539,7 +4576,12 @@ def _agent_output_to_text(agent_output: Any) -> str:def _should_surface_untested_draft_despite_question(ctx: CopilotContext, response_type: str) -> bool:- if response_type != "ASK_QUESTION" or ctx.last_workflow is None or not ctx.last_workflow_yaml:+ if (+ response_type != "ASK_QUESTION"+ or ctx.last_workflow is None+ or not ctx.last_workflow_yaml+ or _is_saved_workflow(ctx, ctx.last_workflow)+ ):return Falsereturn ctx.last_test_ok is None and ctx.last_run_skipped_unbound_credentials@@ -4581,16 +4623,17 @@ def _evaluate_copilot_final_output_policy() -> tuple[OutputPolicyVerdict, str, dict[str, Any]]:action_data, response_type, policy_user_response = _final_output_policy_reply(agent_output)+ candidate_workflow = None if _is_saved_workflow(ctx, ctx.last_workflow) else ctx.last_workflowworkflow_yaml = Noneif response_type == "REPLACE_WORKFLOW" and isinstance(action_data.get("workflow_yaml"), str):workflow_yaml = action_data["workflow_yaml"]- elif isinstance(getattr(ctx, "last_workflow_yaml", None), str):+ elif candidate_workflow is not None and isinstance(ctx.last_workflow_yaml, str):workflow_yaml = ctx.last_workflow_yamlworkflow_attempted = ctx.has_genuine_workflow_attempt()_log_output_policy_parity(ctx,- has_workflow_proposal=bool(workflow_yaml or ctx.last_workflow is not None),+ has_workflow_proposal=bool(workflow_yaml or candidate_workflow is not None),workflow_attempted=workflow_attempted,)surface_untested_draft = _should_surface_untested_draft_despite_question(ctx, response_type)@@ -4598,7 +4641,7 @@ def _evaluate_copilot_final_output_policy(if surface_untested_draft:policy_user_response = _rewrite_failed_test_response(policy_user_response, ctx)updated_workflow_for_kind = (- ctx.last_workflow if ctx.last_workflow is not None else WORKFLOW_PRESENT_SENTINEL if workflow_yaml else None+ candidate_workflow if candidate_workflow is not None else WORKFLOW_PRESENT_SENTINEL if workflow_yaml else None)if surface_untested_draft:output_kind = (@@ -4621,7 +4664,7 @@ def _evaluate_copilot_final_output_policy(user_response=policy_user_response,global_llm_context=action_data.get("global_llm_context"),workflow_yaml=workflow_yaml,- has_workflow_proposal=bool(workflow_yaml or ctx.last_workflow is not None),+ has_workflow_proposal=bool(workflow_yaml or candidate_workflow is not None),workflow_was_persisted=ctx.workflow_persisted,workflow_attempted=workflow_attempted,output_kind=output_kind,@@ -4790,7 +4833,12 @@ def _build_output_policy_blocked_result(local_blocker_signal = ctx.blocker_signal if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal) else Noneblocker_active = local_blocker_signal is not None and local_blocker_signal.renders_final_replypreserved_workflow = (- ctx.last_workflow if ctx.last_workflow is not None and ctx.last_workflow_yaml and not blocker_active else None+ ctx.last_workflow+ if ctx.last_workflow is not None+ and ctx.last_workflow_yaml+ and not blocker_active+ and not _is_saved_workflow(ctx, ctx.last_workflow)+ else None)preserved_workflow_yaml = ctx.last_workflow_yaml if preserved_workflow is not None else Noneoutput_policy_reasons = list(verdict.reason_codes if evaluated_reason_codes is None else evaluated_reason_codes)diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex c846f6c94..3ae0c66e8 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -1368,6 +1368,8 @@ class CopilotContext(AgentContext):private_workflow_settings: dict[str, Any] = field(default_factory=dict, repr=False)last_workflow: Workflow | None = Nonelast_workflow_yaml: str | None = None+ # Copies of the saved workflow a full test loaded this turn; matched by identity, never offered as a proposal.+ saved_workflows_under_test: list[Workflow] = field(default_factory=list)# Always False under staging; ``has_staged_proposal`` carries the signal.workflow_persisted: bool = Falselast_update_block_count: int | None = Nonediff --git a/skyvern/forge/sdk/copilot/model_telemetry.py b/skyvern/forge/sdk/copilot/model_telemetry.pyindex ad18d0915..3e2bcdab2 100644--- a/skyvern/forge/sdk/copilot/model_telemetry.py+++ b/skyvern/forge/sdk/copilot/model_telemetry.py@@ -10,7 +10,7 @@ import timefrom collections.abc import AsyncIterator, Callable, Iteratorfrom dataclasses import dataclass, field, replacefrom types import SimpleNamespace-from typing import Any, Literal, cast, overload+from typing import Any, Literal, TypeVar, cast, overloadfrom urllib.parse import urlparseimport litellm@@ -25,7 +25,7 @@ from agents.models.chatcmpl_converter import Converterfrom agents.models.fake_id import FAKE_RESPONSES_IDfrom agents.models.interface import ModelTracingfrom agents.models.openai_responses import Converter as OpenAIResponsesConverter-from agents.tool import Tool+from agents.tool import FunctionTool, Toolfrom agents.tracing.span_data import GenerationSpanDatafrom agents.tracing.spans import Spanfrom litellm.completion_extras import responses_api_bridge@@ -49,6 +49,7 @@ from skyvern.forge.sdk.api.llm.copilot_model_usage import (normalize_gen_ai_provider,)from skyvern.forge.sdk.api.llm.litellm_pricing import long_prompt_cost_multiplier+from skyvern.forge.sdk.api.llm.utils import is_anthropic_modelfrom skyvern.forge.sdk.copilot.cache_envelope import (CacheableSystemInstructions,ExplicitCacheEnvelope,@@ -611,6 +612,40 @@ def _escape_ref_tool_outputs_for_gemini(return rewritten+_SCHEMA_LITERALS = frozenset({"default", "const", "enum", "examples"})+++def _drop_schema_titles(node: Any) -> Any:+ if isinstance(node, list):+ return [_drop_schema_titles(item) for item in node]+ if not isinstance(node, dict):+ return node+ return {+ key: value if key in _SCHEMA_LITERALS else _drop_schema_titles(value)+ for key, value in node.items()+ if not (key == "title" and isinstance(value, str))+ }+++_ToolT = TypeVar("_ToolT", bound=Tool)+++def drop_tool_schema_titles(tools: list[_ToolT], model: str, model_settings: ModelSettings) -> list[_ToolT]:+ # Pydantic's generated titles carry nothing the property names don't. Offline replay showed no behavior change+ # without them on Claude and terra; gpt-6.1-sol replied less often after a passing test run, so other models keep them.+ if not all(+ is_anthropic_model(candidate) or "gpt-5.6-terra" in candidate+ for candidate in model_chain(model, model_settings)+ ):+ return tools+ return [+ replace(tool, params_json_schema=_drop_schema_titles(tool.params_json_schema))+ if isinstance(tool, FunctionTool)+ else tool+ for tool in tools+ ]++class CopilotLitellmModel(LitellmModel):def __init__(self,@@ -636,7 +671,6 @@ class CopilotLitellmModel(LitellmModel):conversation_id: str | None = None,prompt: Any | None = None,) -> ModelResponse:- attach_tool_surface_to_pending_capture(tools)with _model_call_telemetry_scope(self.next_model_call_index(),model=self.model,@@ -668,7 +702,6 @@ class CopilotLitellmModel(LitellmModel):conversation_id: str | None = None,prompt: Any | None = None,) -> AsyncIterator[TResponseStreamEvent]:- attach_tool_surface_to_pending_capture(tools)with _model_call_telemetry_scope(self.next_model_call_index(),model=self.model,@@ -732,6 +765,8 @@ class CopilotLitellmModel(LitellmModel):prompt: Any | None = None,) -> LiteLLMModelResponse | tuple[Response, AsyncStream[ChatCompletionChunk]]:input = _escape_ref_tool_outputs_for_gemini(input, self.model, model_settings)+ tools = drop_tool_schema_titles(tools, self.model, model_settings)+ attach_tool_surface_to_pending_capture(tools)anchors = stable_prefix_anchors(input) if isinstance(input, list) else []anchor = anchors[-1] if anchors else Noneexplicit_cache_envelope = build_explicit_cache_envelope(diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 422e211ba..c8ce9c30c 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -22,7 +22,6 @@ from typing import Any, Literal, NamedTuple, NotRequired, TypedDictfrom urllib.parse import urlparseimport structlog-import yamlfrom fastapi import HTTPExceptionfrom skyvern.constants import SCRUBBED_VALUE@@ -240,6 +239,7 @@ from skyvern.forge.sdk.copilot.workflow_yaml import (_process_workflow_yaml,runner_code_block_associations,stored_block_code,+ workflow_to_copilot_yaml,)from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.db.enums import BrowserSeedSource@@ -4043,8 +4043,7 @@ async def _run_blocks_and_collect_debug(workflow = snapshot.workflowexecution = _RunExecution(snapshot=snapshot,- workflow_yaml=snapshot.workflow_yaml- or yaml.safe_dump({"workflow_definition": workflow.workflow_definition.model_dump(mode="json")}),+ workflow_yaml=snapshot.workflow_yaml or workflow_to_copilot_yaml(workflow),metadata=copy.deepcopy(ctx.code_artifact_metadata),associations=copy.deepcopy(ctx.runner_code_block_associations_by_label),source_at_start=(@@ -4081,6 +4080,7 @@ async def _run_blocks_and_collect_debug(# Existing verification consumers need the same source this full test selected.ctx.last_workflow = snapshot.workflowctx.last_workflow_yaml = execution.workflow_yaml+ ctx.saved_workflows_under_test.append(snapshot.workflow)for label in block_labels:if not workflow.get_output_parameter(label):diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex e1e25e23f..753276517 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -5961,6 +5961,7 @@ class _CredentialWorkflow(BaseModel):"""Raw definition fixture for credential admission, with snapshot copy semantics."""workflow_id: str = "wf-1"+ workflow_permanent_id: str = "wfp-1"workflow_definition: _CredentialWorkflowDefinitionoutput_labels: set[str]diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex fdab6c418..cafe39c76 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -30,6 +30,7 @@ from skyvern.forge.sdk.copilot.agent import (_prior_run_debug_text,_recorded_build_test_outcome_prompt,_review_projection_for,+ _translate_to_agent_result,)from skyvern.forge.sdk.copilot.build_test_connect_failure import build_test_connect_failure_sentencefrom skyvern.forge.sdk.copilot.build_test_outcome import (@@ -67,9 +68,10 @@ from skyvern.forge.sdk.copilot.build_test_outcome import ()from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig-from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext+from skyvern.forge.sdk.copilot.context import AgentResult, CodeAuthoringRepairContext, CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import _summarize_tool_outputfrom skyvern.forge.sdk.copilot.hooks import CopilotRunHooks+from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKindfrom skyvern.forge.sdk.copilot.output_utils import (_BUILD_TEST_PACKET_MAX_CHARS,_INTERNAL_RUN_OUTCOME_RECORDED_KEY,@@ -82,6 +84,7 @@ from skyvern.forge.sdk.copilot.output_utils import (project_direct_test_handoff_packet_for_llm,sanitize_tool_result_for_llm,)+from skyvern.forge.sdk.copilot.request_policy import RequestPolicyfrom skyvern.forge.sdk.copilot.review_gate import workflow_block_fingerprintsfrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.runtime_authoring_repair import (@@ -8923,3 +8926,154 @@ async def test_an_unprovable_executed_version_is_unavailable_and_never_falls_bac"workflow_id": "wf-1","reason": reason,}+++_SAVED_SIGN_IN_AND_SEARCH_YAML = """+title: sign in and search+workflow_definition:+ version: 2+ parameters:+ - key: login_credentials+ parameter_type: workflow+ workflow_parameter_type: credential_id+ default_value: cred_saved+ - key: search_term+ parameter_type: workflow+ workflow_parameter_type: string+ default_value: widgets+ blocks:+ - block_type: login+ label: sign_in+ url: https://example.com/login+ parameter_keys: [login_credentials]+ next_block_label: search+ - block_type: code+ label: search+ parameter_keys: [login_credentials, search_term]+ code: |+ await page.fill("#q", search_term)+"""++_PROPOSAL_OUTPUT_KINDS = {+ CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL.value,+ CopilotOutputKind.WORKFLOW_UPDATE_PROPOSAL.value,+}+++def _saved_sign_in_ctx() -> CopilotContext:+ return make_copilot_ctx(+ browser_session_id="pbs_chat",+ workflow_copilot_chat_id="chat-1",+ request_policy=RequestPolicy(persisted_workflow_credential_ids=["cred_saved"]),+ )+++async def _blank_browser_retest_of_saved_workflow(+ monkeypatch: pytest.MonkeyPatch, ctx: CopilotContext, status: str+) -> None:+ await install_run_blocks_harness(+ monkeypatch,+ workflow_yaml=_SAVED_SIGN_IN_AND_SEARCH_YAML,+ polled_status=status,+ dispatch_to_worker=True,+ terminal_blocks=[+ terminal_extraction_block("completed", label="sign_in"),+ terminal_extraction_block(status, label="search"),+ ],+ )+ forge_app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final = AsyncMock()+ monkeypatch.setattr(run_execution_module, "_credential_ids_validation_error", AsyncMock(return_value=None))++ async def mint_blank_browser(acquisition_ctx: CopilotContext, *, fresh: bool, **_kwargs: object) -> None:+ acquisition_ctx.browser_session_id = "pbs_run"++ monkeypatch.setattr(run_execution_module, "acquire_build_test_browser_session", mint_blank_browser)+ workflow_service_module.prepare_workflow.return_value.browser_session_id = "pbs_run"+ result = await run_execution_module.run_workflow_end_to_end(ctx)+ assert result["data"]["execution_source"]["source_kind"] == "canonical", result+++async def test_canonical_blank_browser_retest_records_yaml_the_accept_resolver_keeps_bindings(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = _saved_sign_in_ctx()++ await _blank_browser_retest_of_saved_workflow(monkeypatch, ctx, "failed")++ assert ctx.last_workflow is not None and ctx.last_workflow_yaml+ proposal = workflow_copilot_route._build_proposed_workflow_data(+ ctx.last_workflow,+ AgentResult(+ user_response="",+ updated_workflow=ctx.last_workflow,+ workflow_yaml=ctx.last_workflow_yaml,+ global_llm_context=None,+ ),+ )+ resolved = await workflow_copilot_route._resolve_proposal_for_accept(+ proposal, organization_id="org-1", workflow_permanent_id="wfp-1"+ )++ blocks = {block.label: block for block in resolved.workflow_definition.blocks}+ assert blocks["search"].parameter_keys == ["login_credentials", "search_term"]+ assert blocks["sign_in"].parameter_keys == ["login_credentials"]+ assert {parameter.key for parameter in resolved.workflow_definition.parameters} >= {+ "login_credentials",+ "search_term",+ }+ assert ctx.latest_diagnosis_repair_contract is not None+ assert {+ lever.credential_id+ for lever in ctx.latest_diagnosis_repair_contract.levers+ if lever.mechanism == "credential_update"+ } == {"cred_saved"}++[email protected]("run_statuses", [("completed",), ("failed",), ("completed", "failed")])+ ("response_type", "product_action"),+ [("REPLY", None), ("ASK_QUESTION", None), ("REPLY", "test_end_to_end")],+)+async def test_canonical_blank_browser_retests_stage_no_proposal_on_any_exit(+ monkeypatch: pytest.MonkeyPatch, run_statuses: tuple[str, ...], response_type: str, product_action: str | None+) -> None:+ ctx = _saved_sign_in_ctx()++ for status in run_statuses:+ await _blank_browser_retest_of_saved_workflow(monkeypatch, ctx, status)++ failed = run_statuses[-1] == "failed"+ assert ctx.last_workflow is not None and ctx.last_test_ok is not failed+ chat_request = SimpleNamespace(+ workflow_id="wf-1",+ workflow_permanent_id="wfp-1",+ workflow_copilot_chat_id="chat-1",+ workflow_run_id=None,+ workflow_yaml="",+ product_action=product_action,+ )+ normal_exit = await _translate_to_agent_result(+ SimpleNamespace(+ final_output=json.dumps({"type": response_type, "user_response": "I re-ran the saved workflow."}),+ new_items=[],+ ),+ ctx,+ None,+ chat_request,+ "org-1",+ )+ wip_exit = _build_wip_exit_result(ctx, None, default_reply="Stopped.")++ for result in (normal_exit, wip_exit):+ assert (result.updated_workflow, result.workflow_yaml, result.staged_workflow) == (None, None, None)+ assert result.proposal_disposition == "no_proposal"+ assert result.clear_proposed_workflow is False+ assert (result.output_policy_diagnostics or {}).get("final_output_kind") not in _PROPOSAL_OUTPUT_KINDS+ assert ctx.last_workflow is not None+ if failed:+ ctx.saved_workflows_under_test.clear()+ draft_wip_exit = _build_wip_exit_result(ctx, None, default_reply="Stopped.")+ assert draft_wip_exit.updated_workflow is ctx.last_workflow+ assert wip_exit.user_response != draft_wip_exit.user_responsediff --git a/tests/unit/test_copilot_model_telemetry.py b/tests/unit/test_copilot_model_telemetry.pyindex adcbbd2dc..789f65cac 100644--- a/tests/unit/test_copilot_model_telemetry.py+++ b/tests/unit/test_copilot_model_telemetry.py@@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Iteratorfrom dataclasses import dataclassfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom types import SimpleNamespace-from typing import Any, Self, cast+from typing import Any, Literal, Self, castfrom unittest.mock import AsyncMock, MagicMockimport httpx@@ -226,15 +226,13 @@ async def test_model_adapter_attaches_exact_tools_for_nonstream_and_stream_reque) -> None:attached: list[list[Any]] = []- async def fake_get_response(*_args: Any, **_kwargs: Any) -> SimpleNamespace:- return SimpleNamespace(output=[])-- async def fake_stream_response(*_args: Any, **_kwargs: Any) -> AsyncIterator[str]:- yield "done"+ async def fake_acompletion(**kwargs: Any) -> Any:+ if kwargs.get("stream"):+ return cast(AsyncStream[ChatCompletionChunk], _ChunkStream(_stream_chunks()))+ return _completion()monkeypatch.setattr(model_telemetry_module, "attach_tool_surface_to_pending_capture", attached.append)- monkeypatch.setattr(LitellmModel, "get_response", fake_get_response)- monkeypatch.setattr(LitellmModel, "stream_response", fake_stream_response)+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)model = CopilotLitellmModel(model="openai/gpt-5.6", next_model_call_index=lambda: 1)await model.get_response(@@ -260,7 +258,7 @@ async def test_model_adapter_attaches_exact_tools_for_nonstream_and_stream_reque]assert attached == [[_lookup_number], [_lookup_number]]- assert streamed == ["done"]+ assert streamed@pytest.mark.asyncio@@ -280,6 +278,62 @@ async def test_nonstream_capture_preserves_request_and_result(monkeypatch: pytesassert current_model_call_telemetry() is None+@function_tool(strict_mode=False)+def _pick(title: str | None = None, mode: Literal["a", "b"] | None = None) -> str:+ return f"{title}{mode}"+++ ("model_name", "drops_titles"),+ [("anthropic/claude-sonnet-5-5", True), ("azure/gpt-5.6-terra", True), ("gpt-6.1-sol", False)],+)+async def test_sent_tool_schemas_drop_generated_titles_only_on_replay_checked_models(+ monkeypatch: pytest.MonkeyPatch,+ model_name: str,+ drops_titles: bool,+) -> None:+ sent: list[dict[str, Any]] = []+ attached: list[list[Any]] = []++ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse:+ sent.append(kwargs["tools"][0]["function"]["parameters"])+ return _completion()++ async def fake_aresponses(**kwargs: Any) -> ResponsesAPIResponse:+ sent.append(kwargs["tools"][0]["parameters"])+ return _responses_completion()++ monkeypatch.setattr("litellm.acompletion", fake_acompletion)+ monkeypatch.setattr("litellm.aresponses", fake_aresponses)+ monkeypatch.setattr(model_telemetry_module, "attach_tool_surface_to_pending_capture", attached.append)+ original = copy.deepcopy(_pick.params_json_schema)++ await CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1).get_response(+ system_instructions=_CACHED_PROMPT,+ input=[{"role": "user", "content": "Pick one"}],+ model_settings=ModelSettings(temperature=0, include_usage=True, extra_args=_GPT_REASONING),+ tools=[_pick],+ output_schema=None,+ handoffs=[],+ tracing=ModelTracing.DISABLED,+ )++ assert _pick.params_json_schema == original+ assert sent == [+ {+ "properties": {+ "title": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": None},+ "mode": {"anyOf": [{"enum": ["a", "b"], "type": "string"}, {"type": "null"}], "default": None},+ },+ "type": "object",+ }+ if drops_titles+ else original+ ]+ assert [[tool.params_json_schema for tool in tools] for tools in attached] == [sent]++@pytest.mark.asyncioasync def test_direct_gpt56_adds_one_stable_prefix_breakpoint_without_changing_prompt_text(monkeypatch: pytest.MonkeyPatch,