Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Document round-one self-hosted identity QA results (#9060)
Aaron Perez4bf9d1a5bedb
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.pyindex 63b7baec4..3e9f41951 100644--- a/skyvern/core/script_generations/real_skyvern_page_ai.py+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py@@ -23,6 +23,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import (get_org_aware_secondary_llm_api_handler,)from skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_result+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadatafrom skyvern.forge.sdk.cache import extraction_cachefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_prompt_type_handler@@ -887,9 +888,7 @@ class RealSkyvernPageAi(SkyvernPageAi):LOG.info("page.classify: LLM classification result",- classification=classification,- confidence=confidence,- reasoning=reasoning,+ **llm_response_log_metadata(json_response),)if classification in options or classification == "UNKNOWN":@@ -1660,7 +1659,7 @@ class RealSkyvernPageAi(SkyvernPageAi):LOG.warning("ai_extract cache store failed; ignoring", exc_info=True)if context and context.script_mode:- LOG.debug("Extracted information", result=result)+ LOG.debug("Extracted information", **llm_response_log_metadata(result))return resultasync def ai_validate(@@ -1720,14 +1719,12 @@ class RealSkyvernPageAi(SkyvernPageAi):if not result or not isinstance(result, dict):LOG.error("AI locate element failed - invalid result",- result=result,- result_type=type(result).__name__,+ **llm_response_log_metadata(result),prompt=prompt_rendered,)return Noneelement_id = result.get("element_id", None)- confidence = result.get("confidence_float", 0.0)xpath: str | None = Noneif element_id:@@ -1741,17 +1738,14 @@ class RealSkyvernPageAi(SkyvernPageAi):if not xpath:LOG.error("AI locate element failed - no xpath in element data",- element_id=element_id,- result=result,+ **llm_response_log_metadata(result),prompt=prompt_rendered,)return NoneLOG.info("AI locate element result",- element_id=element_id,- xpath=xpath,- confidence=confidence,+ **llm_response_log_metadata(result),prompt=prompt_rendered,)@@ -1807,25 +1801,29 @@ class RealSkyvernPageAi(SkyvernPageAi):)if not json_response or "inferred_actions" not in json_response:- LOG.warning("ai_act: failed to infer action type", prompt=prompt, response=json_response)+ LOG.warning(+ "ai_act: failed to infer action type",+ prompt=prompt,+ **llm_response_log_metadata(json_response),+ )returninferred_actions = json_response.get("inferred_actions", [])if not inferred_actions:- error = json_response.get("error")- LOG.warning("ai_act: no action type inferred", prompt=prompt, error=error)+ LOG.warning(+ "ai_act: no action type inferred",+ prompt=prompt,+ **llm_response_log_metadata(json_response),+ )returnaction_info = inferred_actions[0]action_type = action_info.get("action_type")- confidence = action_info.get("confidence_float", 0.0)LOG.info("ai_act: inferred action type",prompt=prompt,- action_type=action_type,- confidence=confidence,- reasoning=action_info.get("reasoning"),+ **llm_response_log_metadata(json_response),)if not skip_refresh:diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex b3a89e1a8..e2bb4687c 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -131,7 +131,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (LLMResponseMissingActionsError,)from skyvern.forge.sdk.api.llm.ui_tars_llm_caller import UITarsLLMCaller-from skyvern.forge.sdk.api.llm.utils import uses_only_anthropic_models+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata, uses_only_anthropic_modelsfrom skyvern.forge.sdk.api.llm.vertex_cache_manager import get_cache_managerfrom skyvern.forge.sdk.api.llm.yutori_navigator_llm_caller import (YutoriNavigatorLLMCaller,@@ -6933,7 +6933,7 @@ class ForgeAgent:screenshots=scraped_page.screenshots,system_prompt=task.workflow_system_prompt,)- LOG.info("Skyvern response to CUA question", skyvern_response=skyvern_response)+ LOG.info("Skyvern response to CUA question", **llm_response_log_metadata(skyvern_response))cua_question = cua_question or skyvern_response.get("question_or_decision")resp_content = skyvern_response.get("answer")if not resp_content:@@ -7163,7 +7163,7 @@ class ForgeAgent:# Generate response using the LLMCallerresponse_content = await llm_caller.generate_ui_tars_response(step)- LOG.info(f"UI-TARS raw response: {response_content}")+ LOG.info("UI-TARS raw response", **llm_response_log_metadata(response_content))window_dimension = (cast(Resolution, scraped_page.window_dimension)@@ -11116,8 +11116,9 @@ class ForgeAgent:"Parallel verification: termination required, marking task as terminated",step_id=step.step_id,task_id=task.task_id,- reasoning=skyvern_context.mask_multi_field_totp_data(- complete_action.reasoning, task_id=task.task_id+ reasoning_type=type(complete_action.reasoning).__name__,+ reasoning_length=(+ len(complete_action.reasoning) if isinstance(complete_action.reasoning, str) else None),)final_status = step.speculative_original_status or StepStatus.completed@@ -11496,8 +11497,8 @@ class ForgeAgent:"Dropped LLM-returned error codes not in user error_code_mapping",task_id=task.task_id,step_id=step.step_id,- dropped_codes=dropped,- allowed_codes=sorted((task.error_code_mapping or {}).keys()),+ dropped_count=len(dropped),+ allowed_codes_count=len(task.error_code_mapping or {}),)return kept@@ -11985,7 +11986,7 @@ class ForgeAgent:"Step completed and goal achieved, marking task as completed",step_order=step.order,step_retry=step.retry_index,- output=step.output,+ **llm_response_log_metadata(step.output.model_dump() if step.output is not None else None),)last_step = await self.update_step(step, is_last=True)extracted_information = await self.get_extracted_information_for_task(task)@@ -12036,9 +12037,7 @@ class ForgeAgent:"Step completed and terminated by the agent, marking task as terminated",step_order=step.order,step_retry=step.retry_index,- output=skyvern_context.mask_multi_field_totp_data(step.output.model_dump(), task_id=task.task_id)- if step.output is not None and task.task_id in skyvern_context.multi_field_totp_masking_task_ids()- else step.output,+ **llm_response_log_metadata(step.output.model_dump() if step.output is not None else None),)last_step = await self.update_step(step, is_last=True)failure_category = classify_from_failure_reason(failure_reason)@@ -12076,7 +12075,7 @@ class ForgeAgent:"Step completed for the action block, marking task as completed",step_order=step.order,step_retry=step.retry_index,- output=step.output,+ **llm_response_log_metadata(step.output.model_dump() if step.output is not None else None),)last_step = await self.update_step(step, is_last=True)await self.update_task(diff --git a/skyvern/forge/sdk/api/llm/utils.py b/skyvern/forge/sdk/api/llm/utils.pyindex d61844275..63f98321d 100644--- a/skyvern/forge/sdk/api/llm/utils.py+++ b/skyvern/forge/sdk/api/llm/utils.py@@ -277,6 +277,16 @@ def _reassemble_list_response(response: list[Any], allow_action_list_wrap: bool)return None+def llm_response_log_metadata(response: Any) -> dict[str, str | int]:+ """Return content-free metadata suitable for logging an LLM response."""+ metadata: dict[str, str | int] = {"response_type": type(response).__name__}+ if isinstance(response, dict):+ metadata["response_keys_count"] = len(response)+ elif isinstance(response, (list, str)):+ metadata["response_length"] = len(response)+ return metadata++def _coerce_response_to_dict(response: Any, prompt_name: str | None = None) -> dict[str, Any]:"""Ensure parsed LLM responses expose a dict interface to callers."""if isinstance(response, dict):@@ -295,7 +305,7 @@ def _coerce_response_to_dict(response: Any, prompt_name: str | None = None) -> dLOG.info("Recovered single-object response from list-shaped LLM output",response_length=len(response),- recovered_keys=list(reassembled.keys()),+ recovered_keys_count=len(reassembled),prompt_name=prompt_name,)return reassembled@@ -305,7 +315,7 @@ def _coerce_response_to_dict(response: Any, prompt_name: str | None = None) -> d"Parsed LLM response is a list; using first dict element",response_length=len(response),first_item_type=type(response[0]).__name__ if response else None,- first_item_keys=list(first_dict.keys()) if first_dict else None,+ first_item_keys_count=len(first_dict) if first_dict else None,prompt_name=prompt_name,)if first_dict is not None:diff --git a/skyvern/forge/sdk/routes/prompts.py b/skyvern/forge/sdk/routes/prompts.pyindex 72535bd0a..5f9e2b6f0 100644--- a/skyvern/forge/sdk/routes/prompts.py+++ b/skyvern/forge/sdk/routes/prompts.py@@ -13,6 +13,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (InvalidLLMResponseType,LLMProviderError,)+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadatafrom skyvern.forge.sdk.routes.routers import base_routerfrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.prompts import (@@ -119,7 +120,7 @@ async def improve_prompt(use_case=use_case,organization_id=current_org.organization_id,prompt=request.prompt,- improved_prompt=output,+ **llm_response_log_metadata(output),)response = ImprovePromptResponse(diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex ff42f0a59..70473aaaf 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -167,6 +167,7 @@ from skyvern.forge.sdk.api.llm.schema_validator import (resolve_schema_type,validate_schema,)+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadatafrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.artifact.storage.base import get_download_retry_started_at, is_file_from_retry_attemptfrom skyvern.forge.sdk.copilot.code_block_security import SlottedReference, is_inert_slot, slot_template@@ -2981,7 +2982,7 @@ class BaseTaskBlock(Block):organization_id=workflow_run.organization_id,current_retry=current_retry,max_retries=self.max_retries,- task_output=task_output.model_dump_json(),+ **llm_response_log_metadata(task_output.model_dump()),)if not will_retry:output_parameter_value = task_output.model_dump()@@ -11297,7 +11298,7 @@ class TextPromptBlock(Block):except Exception as e:LOG.error("Failed to save TextPromptBlock artifacts", error=e)- LOG.info("TextPromptBlock Received response from LLM", response=response)+ LOG.info("TextPromptBlock Received response from LLM", **llm_response_log_metadata(response))return responseasync def send_prompt_with_schema_retries(@@ -12870,7 +12871,8 @@ class FileUploadBlock(FileDestinationBlock):workflow_run_block_id=workflow_run_block_id,candidate_count=candidate_count,selected_count=selected_count,- reasoning=selection_reasoning,+ reasoning_type=type(selection_reasoning).__name__,+ reasoning_length=len(selection_reasoning) if isinstance(selection_reasoning, str) else None,)await self.record_output_parameter_value(workflow_run_context, workflow_run_id, uploaded_uris)return await self.build_block_result(@@ -16697,7 +16699,8 @@ class FileDownloadBlock(BaseTaskBlock, FileDestinationBlock):workflow_run_block_id=workflow_run_block_id,candidate_count=candidate_count,selected_count=selected_count,- reasoning=selection_reasoning,+ reasoning_type=type(selection_reasoning).__name__,+ reasoning_length=len(selection_reasoning) if isinstance(selection_reasoning, str) else None,)return result@@ -18714,7 +18717,8 @@ def _parse_single_evaluation(LOG.warning("Conditional branch evaluation returned non-boolean result",branch_index=idx,- result=result,+ result_type=type(result).__name__,+ result_length=len(result) if isinstance(result, str) else None,evaluated_result=bool_result,)@@ -19273,8 +19277,8 @@ async def _evaluate_prompt_branch_conditions_batch(block_label=log_label,attempt=attempt,will_retry=attempt + 1 < MAX_PROMPT_BRANCH_EVAL_ATTEMPTS,- error=str(malformed),- raw_output=raw_output_value,+ error_type=type(malformed).__name__,+ **llm_response_log_metadata(raw_output_value),)continue@@ -19298,10 +19302,10 @@ async def _evaluate_prompt_branch_conditions_batch(LOG.info("Conditional branch evaluation results",block_label=log_label,- results=results_array,- llm_rendered_expressions=llm_rendered_expressions,+ results_count=len(results_array),+ llm_rendered_expressions_count=len(llm_rendered_expressions),attempt=attempt,- raw_output=normalized_output,+ **llm_response_log_metadata(normalized_output),)return (results_array, llm_rendered_expressions, extraction_goal, normalized_output)@@ -19309,7 +19313,7 @@ async def _evaluate_prompt_branch_conditions_batch("Conditional branch evaluation failed after retries",block_label=log_label,attempts=MAX_PROMPT_BRANCH_EVAL_ATTEMPTS,- error=str(last_malformed),+ error_type=type(last_malformed).__name__ if last_malformed is not None else None,)raise ConditionalBranchEvaluationError(f"Conditional branch evaluation failed: {last_malformed}")diff --git a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py b/skyvern/forge/sdk/workflow/models/pdf_fill_block.pyindex 7b9ae5827..40ddeb0c1 100644--- a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py+++ b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py@@ -458,7 +458,7 @@ class PdfFillBlock(Block):LOG.warning("PdfFillBlock LLM response missing expected key",expected_key=expected_key,- response_keys=list(parsed.keys()),+ response_keys_count=len(parsed),)return parseddiff --git a/skyvern/forge/sdk/workflow/models/split_pdf_block.py b/skyvern/forge/sdk/workflow/models/split_pdf_block.pyindex 1cab81a11..36d1d2959 100644--- a/skyvern/forge/sdk/workflow/models/split_pdf_block.py+++ b/skyvern/forge/sdk/workflow/models/split_pdf_block.py@@ -171,7 +171,7 @@ class SplitPdfBlock(Block):LOG.warning("SplitPdfBlock LLM response missing expected key",expected_key=expected_key,- response_keys=list(parsed.keys()),+ response_keys_count=len(parsed),)return parseddiff --git a/skyvern/services/error_detection_service.py b/skyvern/services/error_detection_service.pyindex 47beb14ba..d1d583d28 100644--- a/skyvern/services/error_detection_service.py+++ b/skyvern/services/error_detection_service.py@@ -17,6 +17,7 @@ from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codesfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_primary_llm_api_handler+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadatafrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.schemas.tasks import Task@@ -156,7 +157,6 @@ async def _detect_errors_from_page(task_id=task.task_id,step_id=step.step_id,error_count=len(user_defined_errors),- errors=[e.error_code for e in user_defined_errors],)return user_defined_errors@@ -238,13 +238,13 @@ async def _detect_errors_from_context(try:user_defined_error = UserDefinedError.model_validate(error_dict)user_defined_errors.append(user_defined_error)- except Exception:+ except Exception as exc:LOG.warning("Failed to validate user-defined error",task_id=task.task_id,step_id=step.step_id,- error_dict=error_dict,- exc_info=True,+ **llm_response_log_metadata(error_dict),+ error_type=type(exc).__name__,)user_defined_errors, dropped = filter_to_user_defined_codes(user_defined_errors, task.error_code_mapping)@@ -253,8 +253,8 @@ async def _detect_errors_from_context("Dropped LLM-returned error codes not in user error_code_mapping",task_id=task.task_id,step_id=step.step_id,- dropped_codes=dropped,- allowed_codes=sorted((task.error_code_mapping or {}).keys()),+ dropped_count=len(dropped),+ allowed_codes_count=len(task.error_code_mapping or {}),)LOG.info(@@ -262,16 +262,15 @@ async def _detect_errors_from_context(task_id=task.task_id,step_id=step.step_id,error_count=len(user_defined_errors),- errors=[e.error_code for e in user_defined_errors],)return user_defined_errors- except Exception:+ except Exception as exc:LOG.warning("Failed to detect errors from context",task_id=task.task_id,step_id=step.step_id,- exc_info=True,+ error_type=type(exc).__name__,)return []diff --git a/skyvern/services/pdf_import_service.py b/skyvern/services/pdf_import_service.pyindex e56b7b415..be03e230a 100644--- a/skyvern/services/pdf_import_service.py+++ b/skyvern/services/pdf_import_service.py@@ -9,6 +9,7 @@ from fastapi import HTTPExceptionfrom skyvern.config import settingsfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadatafrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.utils.pdf_parser import extract_pdf_filefrom skyvern.schemas.workflows import WorkflowCreateYAMLRequest@@ -215,8 +216,7 @@ class PDFImportService:LOG.info("LLM response received",- response_type=type(response),- response_keys=list(response.keys()) if isinstance(response, dict) else None,+ **llm_response_log_metadata(response),organization_id=organization.organization_id,)@@ -225,8 +225,7 @@ class PDFImportService:if not isinstance(response, dict):LOG.error("LLM returned non-dict response",- response_type=type(response),- response=str(response)[:500],+ **llm_response_log_metadata(response),organization_id=organization.organization_id,)raise HTTPException(status_code=422, detail="LLM returned invalid response format - expected JSON object")@@ -235,7 +234,7 @@ class PDFImportService:if response.get("error") == "not_sop":LOG.info("LLM detected non-SOP content",- reason=response.get("reason"),+ **llm_response_log_metadata(response),organization_id=organization.organization_id,)raise HTTPException(@@ -249,7 +248,7 @@ class PDFImportService:if "workflow_definition" not in response:LOG.error("LLM response missing workflow_definition",- response_keys=list(response.keys()),+ **llm_response_log_metadata(response),organization_id=organization.organization_id,)raise HTTPException(status_code=422, detail="LLM response missing 'workflow_definition' field")@@ -257,7 +256,7 @@ class PDFImportService:if "blocks" not in response.get("workflow_definition", {}):LOG.error("LLM workflow_definition missing blocks",- workflow_def_keys=list(response.get("workflow_definition", {}).keys()),+ workflow_def_keys_count=len(response.get("workflow_definition", {})),organization_id=organization.organization_id,)raise HTTPException(status_code=422, detail="LLM workflow definition missing 'blocks' field")@@ -269,17 +268,15 @@ class PDFImportService:LOG.info("Workflow JSON validated successfully",- title=response.get("title"),+ **llm_response_log_metadata(response),block_count=len(response.get("workflow_definition", {}).get("blocks", [])),organization_id=organization.organization_id,)except Exception as e:LOG.error("Failed to validate workflow request",- error=str(e),error_type=type(e).__name__,organization_id=organization.organization_id,- exc_info=True,)raise HTTPException(status_code=422,diff --git a/skyvern/services/script_reviewer.py b/skyvern/services/script_reviewer.pyindex f849f927b..974e31fe5 100644--- a/skyvern/services/script_reviewer.py+++ b/skyvern/services/script_reviewer.py@@ -26,6 +26,7 @@ from skyvern.core.script_generations.script_validators import (from skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handler+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadatafrom skyvern.forge.sdk.workflow.models.block import get_all_blocksfrom skyvern.forge.sdk.workflow.models.parameter import is_sensitive_workflow_parameterfrom skyvern.schemas.scripts import ScriptBranchHit, ScriptFallbackEpisode@@ -1020,7 +1021,7 @@ class ScriptReviewer:"ScriptReviewer: triage result",episode_id=episode.episode_id,code_fixable=is_code_fixable,- triage_response=response_text[:200] if response_text else None,+ **llm_response_log_metadata(response_text),)return is_code_fixableexcept Exception:@@ -1222,8 +1223,7 @@ class ScriptReviewer:block_label=block_label,attempt=attempt,prompt_name=template,- response_type=type(llm_response).__name__,- response_snippet=str(llm_response)[:200],+ **llm_response_log_metadata(llm_response),)updated_code = self._extract_code_from_response(@@ -1237,7 +1237,7 @@ class ScriptReviewer:block_label=block_label,attempt=attempt,prompt_name=template,- response_snippet=str(llm_response)[:500],+ **llm_response_log_metadata(llm_response),)if attempt >= max_attempts:return Nonediff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 15237afc2..97b7e81dc 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -56,6 +56,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import (get_org_aware_primary_llm_api_handler,get_org_aware_secondary_llm_api_handler,)+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadatafrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.artifact.storage.base import get_download_retry_started_at, is_file_from_retry_attemptfrom skyvern.forge.sdk.core import skyvern_context@@ -1498,7 +1499,7 @@ async def _detect_user_defined_errors(except Exception:LOG.warning("Failed to validate user-defined error",- error_dict=error_dict,+ **llm_response_log_metadata(error_dict),)user_defined_errors, dropped = filter_to_user_defined_codes(user_defined_errors, error_code_mapping)@@ -1507,8 +1508,8 @@ async def _detect_user_defined_errors("Dropped LLM-returned error codes not in user error_code_mapping",task_id=task.task_id,step_id=step.step_id,- dropped_codes=dropped,- allowed_codes=sorted((error_code_mapping or {}).keys()),+ dropped_count=len(dropped),+ allowed_codes_count=len(error_code_mapping or {}),)LOG.info(@@ -1516,7 +1517,6 @@ async def _detect_user_defined_errors(task_id=task.task_id,step_id=step.step_id,error_count=len(user_defined_errors),- errors=[e.error_code for e in user_defined_errors],)return user_defined_errorsdiff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.pyindex aad05eb1f..a6fc494e1 100644--- a/skyvern/services/task_v2_service.py+++ b/skyvern/services/task_v2_service.py@@ -37,6 +37,7 @@ from skyvern.forge.sdk.api.llm.custom_llm_registry import (ensure_custom_llm_model_registered_for_org,)from skyvern.forge.sdk.api.llm.exceptions import EmptyLLMResponseError, InvalidLLMResponseFormat+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadatafrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.hashing import generate_url_hash@@ -513,7 +514,7 @@ async def initialize_task_v2_metadata()# validate- LOG.info(f"Initialized task v2 initial response: {metadata_response}")+ LOG.info("Initialized task v2 initial response", **llm_response_log_metadata(metadata_response))url: str = user_url or metadata_response.get("url", "")if not url:raise UrlGenerationFailure()@@ -1086,7 +1087,7 @@ async def run_task_v2_helper()LOG.info("Task v2 response",- task_v2_response=task_v2_response,+ **llm_response_log_metadata(task_v2_response),iteration=i,current_url=current_url,workflow_run_id=workflow_run_id,@@ -1161,12 +1162,12 @@ async def run_task_v2_helper(return workflow, workflow_run, task_v2if not plan:- LOG.warning("No plan found in task v2 response", task_v2_response=task_v2_response)+ LOG.warning("No plan found in task v2 response", **llm_response_log_metadata(task_v2_response))continue# parse task v2 response and run the next taskif not task_type:- LOG.error("No task type found in task v2 response", task_v2_response=task_v2_response)+ LOG.error("No task type found in task v2 response", **llm_response_log_metadata(task_v2_response))task_v2 = await mark_task_v2_as_failed(task_v2_id=task_v2_id,workflow_run_id=workflow_run_id,@@ -1424,10 +1425,10 @@ async def run_task_v2_helper()LOG.info("Task v2 completion check response",- completion_resp=completion_resp,+ **llm_response_log_metadata(completion_resp),iteration=i,workflow_run_id=workflow_run_id,- task_history=task_history,+ task_history_count=len(task_history),)user_goal_achieved = completion_resp.get("user_goal_achieved", False)should_terminate = completion_resp.get("should_terminate", False)@@ -1449,7 +1450,7 @@ async def run_task_v2_helper("User goal achieved according to the task v2 completion check",iteration=i,workflow_run_id=workflow_run_id,- completion_resp=completion_resp,+ **llm_response_log_metadata(completion_resp),)lost_task_v2 = await _fail_task_v2_after_browser_loss(task_v2_id, workflow_run_id, organization_id, completed_block=(block, block_result)@@ -1901,7 +1902,7 @@ async def _generate_loop_task(prompt_name="task_v2_generate_task_block",system_prompt=task_v2.workflow_system_prompt,)- LOG.info("Task in loop metadata response", task_in_loop_metadata_response=task_in_loop_metadata_response)+ LOG.info("Task in loop metadata response", **llm_response_log_metadata(task_in_loop_metadata_response))navigation_goal = task_in_loop_metadata_response.get("navigation_goal")data_extraction_goal = task_in_loop_metadata_response.get("data_extraction_goal")data_extraction_schema = task_in_loop_metadata_response.get("data_schema")@@ -2021,7 +2022,7 @@ async def _generate_extraction_task(raise last_exc # type: ignore[misc]assert generate_extraction_task_response is not None- LOG.info("Data extraction response", data_extraction_response=generate_extraction_task_response)+ LOG.info("Data extraction response", **llm_response_log_metadata(generate_extraction_task_response))# create OutputParameter for the data_extraction blockdata_schema: dict[str, Any] | list | None = generate_extraction_task_response.get("schema")@@ -2134,7 +2135,11 @@ async def _generate_compute_task(plan: str,task_history: list[dict] | None = None,) -> tuple[CodeBlock, list[BLOCK_YAML_TYPES], list[PARAMETER_YAML_TYPES]]:- LOG.info("Generating compute task", plan=plan)+ LOG.info(+ "Generating compute task",+ plan_type=type(plan).__name__,+ plan_length=len(plan) if isinstance(plan, str) else None,+ )context = skyvern_context.ensure_context()gathered = [record for record in (task_history or []) if record.get("extracted_data") is not None]if not gathered:@@ -2647,7 +2652,7 @@ def _get_extracted_data_from_block_result(if not isinstance(inner_loop_output, list):LOG.warning("Inner loop output is not a list",- inner_loop_output=inner_loop_output,+ **llm_response_log_metadata(inner_loop_output),task_v2_id=task_v2_id,workflow_run_id=workflow_run_id,workflow_run_block_id=block_result.workflow_run_block_id,@@ -2657,7 +2662,7 @@ def _get_extracted_data_from_block_result(if not isinstance(inner_output, dict):LOG.warning("inner output is not a dict",- inner_output=inner_output,+ **llm_response_log_metadata(inner_output),task_v2_id=task_v2_id,workflow_run_id=workflow_run_id,workflow_run_block_id=block_result.workflow_run_block_id,@@ -2667,7 +2672,7 @@ def _get_extracted_data_from_block_result(if not isinstance(output_value, dict):LOG.warning("output_value is not a dict",- output_value=output_value,+ **llm_response_log_metadata(output_value),task_v2_id=task_v2_id,workflow_run_id=workflow_run_id,workflow_run_block_id=block_result.workflow_run_block_id,@@ -2814,7 +2819,7 @@ def _parse_task_v2_summary_response(resp: Any) -> tuple[str | None, Any]:}LOG.info("Recovered task_v2 summary output from non-envelope response",- recovered_keys=sorted(output.keys()),+ recovered_keys_count=len(output),)return description, output@@ -2851,7 +2856,11 @@ async def _generate_task_v2_deliverable(prompt_name="task_v2_summary",system_prompt=task_v2.workflow_system_prompt,)- LOG.info("Task v2 summary response", task_v2_summary_resp=task_v2_summary_resp, is_partial=is_partial)+ LOG.info(+ "Task v2 summary response",+ **llm_response_log_metadata(task_v2_summary_resp),+ is_partial=is_partial,+ )summary_description, summarized_output = _parse_task_v2_summary_response(task_v2_summary_resp)await app.DATABASE.observer.update_thought(diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex e511c1dbd..2dea4e2f1 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -130,6 +130,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import ()from skyvern.forge.sdk.api.llm.exceptions import BaseLLMError, LLMProviderErrorfrom skyvern.forge.sdk.api.llm.schema_validator import extraction_shape_matches, validate_and_fill_extraction_result+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadatafrom skyvern.forge.sdk.artifact.storage.base import get_download_retry_started_at, is_file_from_retry_attemptfrom skyvern.forge.sdk.browser_action_preflight import preflight_action, preflight_derived_actionfrom skyvern.forge.sdk.cache import extraction_cache, extraction_shadow@@ -11513,13 +11514,13 @@ async def handle_terminate_action(extracted_errors = await extract_user_defined_errors(task=task, step=step, scraped_page=scraped_page, reasoning=action.reasoning)- except Exception:+ except Exception as exc:LOG.warning("extract_user_defined_errors failed, using errors from action reasoning",task_id=task.task_id,step_id=step.step_id,- action_errors=action.errors,- exc_info=True,+ action_error_count=len(action.errors),+ error_type=type(exc).__name__,)else:skyvern_codes = {error.error_code for error in skyvern_errors}@@ -13466,7 +13467,7 @@ async def input_or_auto_complete_input(if input_or_select_context.is_search_bar:LOG.info("Stop generating potential values for the auto-completion since it's a search bar",- context=input_or_select_context,+ **llm_response_log_metadata(input_or_select_context.model_dump()),)return None@@ -13620,7 +13621,7 @@ async def input_or_auto_complete_input(LOG.info("Auto completion didn't finish, this might leave the input value to be empty.",sampling=True,- context=input_or_select_context,+ **llm_response_log_metadata(input_or_select_context.model_dump()),attempts=attempt_timings,total_seconds=round(_elapsed_seconds(), 1),)@@ -13845,7 +13846,7 @@ async def sequentially_select_from_dropdown(if not force_select and input_or_select_context.is_search_bar:LOG.info("Exit custom selection mode since it's a non-force search bar",- context=input_or_select_context,+ **llm_response_log_metadata(input_or_select_context.model_dump()),)return None@@ -15422,8 +15423,7 @@ async def select_from_emerging_elements(LOG.info("LLM response for the matched element",sampling=True,- matched_value=value,- response=json_response,+ **llm_response_log_metadata(json_response),)# Check the no-match shape before ``ActionType`` coercion — coercing an empty@@ -15861,8 +15861,7 @@ async def select_from_dropdown(LOG.info("LLM response for the matched element",sampling=True,- matched_value=value,- response=json_response,+ **llm_response_log_metadata(json_response),)# Check the no-match shape before ``ActionType`` coercion — coercing an empty@@ -16456,7 +16455,7 @@ async def normal_select()LOG.debug("Parsed input/select context",- context=input_or_select_context,+ **llm_response_log_metadata(input_or_select_context.model_dump()),)select_options_result = await skyvern_element.refresh_select_options()@@ -17300,7 +17299,7 @@ async def _get_input_or_select_context(original_response_type=type(json_response).__name__,original_response_length=len(json_response) if json_response else 0,first_item_type=type(json_response[0]).__name__ if json_response else None,- first_item_keys=list(json_response[0].keys())+ first_item_keys_count=len(json_response[0])if json_response and isinstance(json_response[0], dict)else None,)@@ -17310,7 +17309,7 @@ async def _get_input_or_select_context(input_or_select_context = InputOrSelectContext.model_validate(json_response)LOG.debug("Parsed input/select context",- context=input_or_select_context,+ **llm_response_log_metadata(input_or_select_context.model_dump()),)return input_or_select_context@@ -17385,8 +17384,8 @@ async def extract_user_defined_errors("Dropped LLM-returned error codes not in user error_code_mapping",task_id=task.task_id,step_id=step.step_id,- dropped_codes=dropped,- allowed_codes=sorted((task.error_code_mapping or {}).keys()),+ dropped_count=len(dropped),+ allowed_codes_count=len(task.error_code_mapping or {}),)if not kept and reasoning:return _match_user_defined_error_from_reasoning(task=task, step=step, reasoning=reasoning)diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.pyindex 5fcfe7e26..4b54f2218 100644--- a/skyvern/webeye/actions/parse_actions.py+++ b/skyvern/webeye/actions/parse_actions.py@@ -19,6 +19,7 @@ from skyvern.exceptions import (from skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_primary_llm_api_handler+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadatafrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.schemas.tasks import Task@@ -247,7 +248,10 @@ def parse_action(if key:is_single_char = len(key) == 1 and (key.isdigit() or key.isalpha())if key not in allowed_keys and not is_single_char:- LOG.warning("KEYPRESS action has unsupported key, skipping action", key=key)+ LOG.warning(+ "KEYPRESS action has unsupported key, skipping action",+ **llm_response_log_metadata(key),+ )return NullAction(**base_action_dict)keys = [key]else:@@ -262,7 +266,10 @@ def parse_action(base_action_dict["skyvern_element_data"] = Nonedirection = action.get("direction", "down").lower()if direction not in ("up", "down"):- LOG.warning("SCROLL action has unexpected direction, defaulting to down", direction=direction)+ LOG.warning(+ "SCROLL action has unexpected direction, defaulting to down",+ **llm_response_log_metadata(direction),+ )direction = "down"if direction == "up":scroll_x = 0@@ -281,7 +288,7 @@ def parse_action(except (TypeError, ValueError):LOG.warning("CLOSE_PAGE action returned with a non-integer tab_index, closing the current tab instead",- raw_action=action,+ **llm_response_log_metadata(action),)return ClosePageAction(**base_action_dict)return ClosePageAction(**base_action_dict, tab_index=tab_index)@@ -291,7 +298,10 @@ def parse_action(base_action_dict["skyvern_element_hash"] = Nonebase_action_dict["skyvern_element_data"] = Noneif not data_extraction_goal:- LOG.warning("EXTRACT action returned without a data extraction goal, skipping action", raw_action=action)+ LOG.warning(+ "EXTRACT action returned without a data extraction goal, skipping action",+ **llm_response_log_metadata(action),+ )return NullAction(**base_action_dict)return ExtractAction(**base_action_dict,@@ -305,18 +315,27 @@ def parse_action(base_action_dict["skyvern_element_data"] = Noneurl = action.get("url")if not url or not isinstance(url, str):- LOG.warning("GOTO_URL action returned without a url, skipping action", raw_action=action)+ LOG.warning(+ "GOTO_URL action returned without a url, skipping action",+ **llm_response_log_metadata(action),+ )return NullAction(**base_action_dict)try:validated_url = prepend_scheme_and_validate_url(url.strip())except InvalidUrl:- LOG.warning("GOTO_URL action returned with an invalid url, skipping action", url=url)+ LOG.warning(+ "GOTO_URL action returned with an invalid url, skipping action",+ **llm_response_log_metadata(url),+ )return NullAction(**base_action_dict)# LLM-generated navigation targets can be steered by page content, so block# private/loopback/link-local hosts the same way the task URL boundary does.host = urlparse(validated_url).hostnameif not host or is_blocked_host(host, resolve_dns=False):- LOG.warning("GOTO_URL action targets a blocked host, skipping action", url=validated_url)+ LOG.warning(+ "GOTO_URL action targets a blocked host, skipping action",+ **llm_response_log_metadata(validated_url),+ )return NullAction(**base_action_dict)return GotoUrlAction(**base_action_dict, url=validated_url)@@ -328,28 +347,43 @@ def parse_action(if action_type == ActionType.NEW_TAB:url = action.get("url")if not url or not isinstance(url, str):- LOG.warning("NEW_TAB action returned without a url, skipping action", raw_action=action)+ LOG.warning(+ "NEW_TAB action returned without a url, skipping action",+ **llm_response_log_metadata(action),+ )return NullAction(**base_action_dict)try:validated_url = prepend_scheme_and_validate_url(url.strip())except InvalidUrl:- LOG.warning("NEW_TAB action returned with an invalid url, skipping action", url=url)+ LOG.warning(+ "NEW_TAB action returned with an invalid url, skipping action",+ **llm_response_log_metadata(url),+ )return NullAction(**base_action_dict)host = urlparse(validated_url).hostnameif not host or is_blocked_host(host, resolve_dns=False):- LOG.warning("NEW_TAB action targets a blocked host, skipping action", url=validated_url)+ LOG.warning(+ "NEW_TAB action targets a blocked host, skipping action",+ **llm_response_log_metadata(validated_url),+ )return NullAction(**base_action_dict)return NewTabAction(**base_action_dict, url=validated_url)if action_type == ActionType.SWITCH_TAB:raw_index = action.get("tab_index")if raw_index is None:- LOG.warning("SWITCH_TAB action returned without a tab_index, skipping action", raw_action=action)+ LOG.warning(+ "SWITCH_TAB action returned without a tab_index, skipping action",+ **llm_response_log_metadata(action),+ )return NullAction(**base_action_dict)try:tab_index = int(raw_index)except (TypeError, ValueError):- LOG.warning("SWITCH_TAB action returned with a non-integer tab_index, skipping action", raw_action=action)+ LOG.warning(+ "SWITCH_TAB action returned with a non-integer tab_index, skipping action",+ **llm_response_log_metadata(action),+ )return NullAction(**base_action_dict)return SwitchTabAction(**base_action_dict, tab_index=tab_index)@@ -392,32 +426,31 @@ def parse_actions(LOG.warning("Agent decided to terminate",task_id=task.task_id,- llm_response=json_response,- reasoning=action_instance.reasoning,- actions=actions,+ **llm_response_log_metadata(json_response),+ action_type=type(action_instance).__name__,)actions.append(action_instance)- except UnsupportedActionType:+ except UnsupportedActionType as exc:LOG.error("Unsupported action type when parsing actions",task_id=task.task_id,- raw_action=action,- exc_info=True,+ error_type=type(exc).__name__,+ **llm_response_log_metadata(action),)- except (ValidationError, ValueError):+ except (ValidationError, ValueError) as exc:LOG.warning("Invalid action",task_id=task.task_id,- raw_action=action,- exc_info=True,+ error_type=type(exc).__name__,+ **llm_response_log_metadata(action),)- except Exception:+ except Exception as exc:LOG.error("Failed to marshal action",task_id=task.task_id,- raw_action=action,- exc_info=True,+ error_type=type(exc).__name__,+ **llm_response_log_metadata(action),)if non_object_entries:@@ -427,7 +460,6 @@ def parse_actions(skipped_count=len(non_object_entries),raw_action_count=len(json_response),parsed_action_count=len(actions),- sample_entries=[str(entry)[:200] for entry in non_object_entries[:3]],)############################ This part of code might not be needed ############################@@ -442,8 +474,8 @@ def parse_actions(LOG.warning("Missing elements in scraped page",task_id=task.task_id,- missing_element_ids=missing_element_ids,- all_element_ids=all_element_ids,+ missing_element_ids_count=len(missing_element_ids),+ all_element_ids_count=len(all_element_ids),)############################ This part of code might not be needed ############################return actions@@ -467,13 +499,15 @@ async def parse_cua_actions(if idx < len(reasonings):try:reasoning = reasonings[idx].summary[0].text- except Exception:- LOG.exception(+ except Exception as exc:+ LOG.error("Failed to parse reasoning",task_id=task.task_id,step_id=step.step_id,step_order=step.order,action_order=idx,+ error_type=type(exc).__name__,+ **llm_response_log_metadata(reasonings[idx]),)match action_type:@@ -547,7 +581,7 @@ async def parse_cua_actions(step_id=step.step_id,step_order=step.order,action_order=idx,- whole_path=whole_path,+ **llm_response_log_metadata(whole_path),)action = WaitAction(seconds=5,@@ -579,13 +613,15 @@ async def parse_cua_actions(action.step_order = step.orderaction.action_order = idxactions.append(action)- except Exception:- LOG.exception(+ except Exception as exc:+ LOG.error("Failed to parse action",task_id=task.task_id,step_id=step.step_id,step_order=step.order,action_order=idx,+ error_type=type(exc).__name__,+ **llm_response_log_metadata(cua_action),)breakif not actions:@@ -595,7 +631,7 @@ async def parse_cua_actions(step_id=step.step_id,organization_id=task.organization_id,workflow_run_id=task.workflow_run_id,- response=response.model_dump(),+ **llm_response_log_metadata(response.model_dump()),)reasoning = reasonings[0].summary[0].text if reasonings and reasonings[0].summary else Noneassistant_message = assistant_messages[0].content[0].text if assistant_messages else None@@ -615,13 +651,18 @@ async def parse_anthropic_actions() -> list[Action]:tool_calls = [block for block in assistant_content if block["type"] == "tool_use" and block["name"] == "computer"]reasonings = [block for block in assistant_content if block["type"] == "thinking"]- LOG.info("Anthropic tool calls", tool_calls=tool_calls, reasonings=reasonings, assistant_content=assistant_content)+ LOG.info(+ "Anthropic tool calls",+ tool_call_count=len(tool_calls),+ reasoning_count=len(reasonings),+ assistant_content_count=len(assistant_content),+ )if len(reasonings) > 1:LOG.warning("Anthropic CUA: multiple reasonings in assistant content",task_id=task.task_id,step_id=step.step_id,- assistant_content=assistant_content,+ assistant_content_count=len(assistant_content),)reasoning = reasonings[0]["thinking"] if reasonings else Noneidx = 0@@ -640,7 +681,8 @@ async def parse_anthropic_actions(if not coordinate:LOG.warning("Anthropic CUA error: mouse move action has no coordinate",- tool_call=tool_call,+ tool_call_type=type(tool_call).__name__,+ tool_call_keys_count=len(tool_call),)idx += 1continue@@ -677,7 +719,8 @@ async def parse_anthropic_actions(if not coordinate:LOG.warning("Anthropic CUA error: left click action has no coordinate and it doesn't have mouse_move before it",- tool_call=tool_call,+ tool_call_type=type(tool_call).__name__,+ tool_call_keys_count=len(tool_call),)idx += 1continue@@ -719,7 +762,8 @@ async def parse_anthropic_actions(if not text:LOG.warning("Anthropic CUA error: type action has no text",- tool_call=tool_call,+ tool_call_type=type(tool_call).__name__,+ tool_call_keys_count=len(tool_call),)idx += 1continue@@ -744,7 +788,8 @@ async def parse_anthropic_actions(if not text:LOG.warning("Anthropic CUA error: key action has no text",- tool_call=tool_call,+ tool_call_type=type(tool_call).__name__,+ tool_call_keys_count=len(tool_call),)idx += 1continue@@ -811,7 +856,8 @@ async def parse_anthropic_actions(else:LOG.warning("Anthropic CUA error: unsupported scroll direction",- tool_call=tool_call,+ tool_call_type=type(tool_call).__name__,+ tool_call_keys_count=len(tool_call),)idx += 1continue@@ -867,12 +913,19 @@ async def parse_anthropic_actions(coordinate = tool_call_input.get("coordinate")start_coordinate = tool_call_input.get("start_coordinate")LOG.info(- "Anthropic CUA left click drag action", coordinate=coordinate, start_coordinate=start_coordinate+ "Anthropic CUA left click drag action",+ coordinate_type=type(coordinate).__name__,+ coordinate_length=len(coordinate) if isinstance(coordinate, (list, tuple, str)) else None,+ start_coordinate_type=type(start_coordinate).__name__,+ start_coordinate_length=(+ len(start_coordinate) if isinstance(start_coordinate, (list, tuple, str)) else None+ ),)if not coordinate or not start_coordinate:LOG.warning("Anthropic CUA error: left click drag action has no coordinate or start coordinate",- tool_call=tool_call,+ tool_call_type=type(tool_call).__name__,+ tool_call_keys_count=len(tool_call),)idx += 1continue@@ -921,15 +974,18 @@ async def parse_anthropic_actions(else:LOG.error("Anthropic CUA error: unsupported action",- tool_call=tool_call,+ tool_call_type=type(tool_call).__name__,+ tool_call_keys_count=len(tool_call),)idx += 1- except Exception:- LOG.exception(+ except Exception as exc:+ LOG.error("Anthropic CUA error: failed to parse action",task_id=task.task_id,step_id=step.step_id,- tool_call=tool_call,+ tool_call_type=type(tool_call).__name__,+ tool_call_keys_count=len(tool_call),+ error_type=type(exc).__name__,)breakif not actions:@@ -976,9 +1032,8 @@ async def generate_cua_fallback_actions(prompt_name="cua-fallback-action",step=step,)- LOG.info("Fallback action response", action_response=action_response)+ LOG.info("Fallback action response", **llm_response_log_metadata(action_response))skyvern_action_type = action_response.get("action")- useful_information = action_response.get("useful_information")# use 'other' action as fallback in the 'cua-fallback-action' prompt# it can avoid LLM returning unreasonable actions, and fallback to use 'wait' action in agent instead@@ -992,9 +1047,7 @@ async def generate_cua_fallback_actions("Updating task with useful information",task_id=task.task_id,organization_id=task.organization_id,- useful_information=useful_information,- assistant_message=assistant_message,- reasoning=reasoning,+ **llm_response_log_metadata(action_response),)await app.DATABASE.tasks.update_task(task.task_id,@@ -1203,12 +1256,13 @@ async def parse_ui_tars_actions(action = _create_ui_tars_action(parsed_action, task, step, browser_window_dimension, idx)if action:actions.append(action)- except Exception:- LOG.exception(+ except Exception as exc:+ LOG.error("Failed to create UI-TARS action",task_id=task.task_id,step_id=step.step_id,- parsed_action=parsed_action,+ error_type=type(exc).__name__,+ **llm_response_log_metadata(parsed_action),)continue@@ -1217,17 +1271,18 @@ async def parse_ui_tars_actions("No valid actions generated from UI-TARS response",task_id=task.task_id,step_id=step.step_id,- response_preview=response_content[:200],+ **llm_response_log_metadata(response_content),)return actions- except Exception:- LOG.exception(+ except Exception as exc:+ LOG.error("Failed to parse UI-TARS actions",task_id=task.task_id,step_id=step.step_id,- response_content=response_content[:200],+ error_type=type(exc).__name__,+ **llm_response_log_metadata(response_content),)return []@@ -1307,11 +1362,11 @@ def _parse_ui_tars_response(response_content: str, browser_window_dimension: Resparsed_action["thought"] = thoughtparsed_action["browser_window_dimension"] = browser_window_dimensionparsed_actions.append(parsed_action)- except Exception:+ except Exception as exc:LOG.warning("Failed to parse individual UI-TARS action",- action_str=action_str,- exc_info=True,+ error_type=type(exc).__name__,+ **llm_response_log_metadata(action_str),)continue@@ -1351,8 +1406,12 @@ def _parse_single_action(action_str: str) -> dict[str, Any] | None:"action_inputs": action_inputs,}- except Exception:- LOG.debug(f"Failed to parse action string: {action_str}", exc_info=True)+ except Exception as exc:+ LOG.debug(+ "Failed to parse action string",+ error_type=type(exc).__name__,+ **llm_response_log_metadata(action_str),+ )return None@@ -1508,7 +1567,10 @@ def _create_ui_tars_action()else:- LOG.warning(f"Unsupported UI-TARS action type: {action_type}")+ LOG.warning(+ "Unsupported UI-TARS action type",+ **llm_response_log_metadata(action_type),+ )return None@@ -1581,6 +1643,10 @@ def _extract_ui_tars_coordinates(box_str: str, browser_window_dimension: Resolutelse:return None, None- except Exception:- LOG.debug(f"Failed to parse UI-TARS coordinates: {box_str}", exc_info=True)+ except Exception as exc:+ LOG.debug(+ "Failed to parse UI-TARS coordinates",+ error_type=type(exc).__name__,+ **llm_response_log_metadata(box_str),+ )return None, Nonediff --git a/tests/unit/test_actions.py b/tests/unit/test_actions.pyindex 0f0978f15..b2947f548 100644--- a/tests/unit/test_actions.py+++ b/tests/unit/test_actions.py@@ -1,7 +1,7 @@from datetime import datetimefrom pathlib import Pathfrom typing import Any-from unittest.mock import MagicMock+from unittest.mock import AsyncMock, MagicMockimport pytestfrom pydantic import TypeAdapter, ValidationError@@ -47,7 +47,13 @@ from skyvern.webeye.actions.actions import (WebAction,)from skyvern.webeye.actions.models import DetailedAgentStepOutput-from skyvern.webeye.actions.parse_actions import parse_action, parse_actions+from skyvern.webeye.actions.parse_actions import (+ parse_action,+ parse_actions,+ parse_anthropic_actions,+ parse_cua_actions,+ parse_ui_tars_actions,+)def _mock_scraped_page() -> MagicMock:@@ -57,6 +63,115 @@ def _mock_scraped_page() -> MagicMock:return page+async def test_parse_anthropic_actions_does_not_log_model_response_content(monkeypatch: pytest.MonkeyPatch) -> None:+ sentinel = "private-anthropic-response-sentinel"+ task = MagicMock(task_id="tsk_private", organization_id="org_private", workflow_run_id=None)+ step = MagicMock(step_id="stp_private", order=0)+ assistant_content = [+ {"type": "thinking", "thinking": sentinel},+ {"type": "thinking", "thinking": sentinel},+ {+ "type": "tool_use",+ "name": "computer",+ "id": "tool_private",+ "input": {"action": "mouse_move", "private": sentinel},+ },+ ]+ fallback = AsyncMock(return_value=[])+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.generate_cua_fallback_actions", fallback)++ with capture_logs() as events:+ actions = await parse_anthropic_actions(task, step, assistant_content, None, None)++ assert actions == []+ assert sentinel not in repr(events)+ assert fallback.await_count == 1+++def test_parse_actions_does_not_log_malformed_response_content(monkeypatch: pytest.MonkeyPatch) -> None:+ sentinel = "private-malformed-action-sentinel"+ context = MagicMock(totp_codes={})+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.skyvern_context.ensure_context", lambda: context)+ task = MagicMock(+ task_id="tsk_private",+ organization_id="org_private",+ workflow_run_id=None,+ data_extraction_goal=None,+ extracted_information_schema=None,+ )++ with capture_logs() as events:+ actions = parse_actions(+ task,+ "stp_private",+ 0,+ _mock_scraped_page(),+ [+ {"action_type": "INPUT_TEXT", "id": "e1", "reasoning": sentinel},+ {"action_type": "SWITCH_TAB", "tab_index": sentinel},+ sentinel,+ ],+ )++ assert len(actions) == 1+ assert isinstance(actions[0], NullAction)+ assert sentinel not in repr(events)+ assert not any("exc_info" in event for event in events)+++async def test_parse_cua_actions_does_not_log_malformed_response_content(monkeypatch: pytest.MonkeyPatch) -> None:+ sentinel = "private-cua-action-sentinel"+ task = MagicMock(task_id="tsk_private", organization_id="org_private", workflow_run_id=None)+ step = MagicMock(step_id="stp_private", order=0)+ response = MagicMock()+ response.output = [MagicMock(type="computer_call", action=MagicMock(type=sentinel))]+ response.model_dump.return_value = {"output": [{"type": "computer_call", "action": {"type": sentinel}}]}+ fallback = AsyncMock(return_value=[])+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.generate_cua_fallback_actions", fallback)++ with capture_logs() as events:+ actions = await parse_cua_actions(task, step, response)++ assert actions == []+ assert sentinel not in repr(events)+ assert not any("exc_info" in event for event in events)+++ ("response_content", "sentinel", "expected_event"),+ [+ ("Action: private_ui_tars_sentinel((", "private_ui_tars_sentinel", "Failed to parse action string"),+ ("Action: private_ui_tars_sentinel()", "private_ui_tars_sentinel", "Unsupported UI-TARS action type"),+ (+ "Action: click(start_box='private-ui-tars-sentinel')",+ "private-ui-tars-sentinel",+ "Failed to parse UI-TARS coordinates",+ ),+ ],+ ids=["malformed-action", "unsupported-action-type", "malformed-coordinates"],+)+async def test_parse_ui_tars_actions_does_not_log_malformed_response_content(+ response_content: str,+ sentinel: str,+ expected_event: str,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ task = MagicMock(task_id="tsk_private", organization_id="org_private", workflow_run_id=None)+ step = MagicMock(step_id="stp_private", order=0)+ logger = MagicMock()+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.LOG", logger)++ actions = await parse_ui_tars_actions(task, step, response_content, {"width": 1280, "height": 720})++ assert actions == []+ assert expected_event in repr(logger.mock_calls)+ assert sentinel not in repr(logger.mock_calls)+ assert "exc_info" not in repr(logger.mock_calls)++def test_sdk_input_text_action_type_constant_matches_sdk_enum() -> None:assert SDK_INPUT_TEXT_ACTION_TYPE == SdkActionType.AI_INPUT_TEXT.valuediff --git a/tests/unit/test_background_task_executor.py b/tests/unit/test_background_task_executor.pyindex 632f30799..c1c62b978 100644--- a/tests/unit/test_background_task_executor.py+++ b/tests/unit/test_background_task_executor.py@@ -2627,7 +2627,6 @@ async def test_retry_resumer_recovers_attempt_lookup_failure_on_next_sweep(read_run.side_effect = Nonedispatch.side_effect = real_entryawait executor._recover_pending_retries_once(resume_fresh=False)- assert key in executor._scheduled_retry_resumesawait asyncio.gather(*executor._background_tasks)execute.assert_awaited_once()assert execute.await_args.kwargs["attempt_number"] == 2diff --git a/tests/unit/test_error_detection_service.py b/tests/unit/test_error_detection_service.pyindex a62028d53..5b38c1da1 100644--- a/tests/unit/test_error_detection_service.py+++ b/tests/unit/test_error_detection_service.py@@ -8,6 +8,7 @@ from datetime import datetimefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest+from structlog.testing import capture_logsfrom skyvern.forge.sdk.models import Step, StepStatusfrom skyvern.services.error_detection_service import detect_user_defined_errors_for_task@@ -149,6 +150,32 @@ async def test_detect_errors_no_browser_state(mock_step):assert detected_errors[0].error_code == "payment_failed"+async def test_context_detection_does_not_log_malformed_llm_error_payload(mock_step):+ now = datetime.now()+ organization = make_organization(now)+ task = make_task(now, organization, error_code_mapping={"payment_failed": "Credit card was declined"})+ sentinel = "private-malformed-error-sentinel"++ with patch("skyvern.services.error_detection_service.app") as mock_app, capture_logs() as events:+ mock_app.EXTRACTION_LLM_API_HANDLER = AsyncMock(+ return_value={+ "errors": [+ {"error_code": sentinel, "reasoning": sentinel, "confidence_float": sentinel},+ ]+ }+ )+ detected = await detect_user_defined_errors_for_task(+ task=task, step=mock_step, browser_state=None, failure_reason=None+ )++ assert detected == []+ event = next(event for event in events if event["event"] == "Failed to validate user-defined error")+ assert event["error_type"] == "ValidationError"+ assert "exc_info" not in event+ assert sentinel not in repr(event)++@pytest.mark.asyncioasync def test_detect_errors_no_working_page(mock_step):"""Test that detection uses context-based method when there's no working page."""diff --git a/tests/unit/test_file_upload_prompt.py b/tests/unit/test_file_upload_prompt.pyindex f44aa33e0..8fdac307e 100644--- a/tests/unit/test_file_upload_prompt.py+++ b/tests/unit/test_file_upload_prompt.py@@ -410,11 +410,13 @@ async def test_prompt_selecting_zero_files_completes_and_records_empty_output("workflow_run_block_id": "workflow-run-block-id","candidate_count": 3,"selected_count": 0,- "reasoning": "No candidates match.",+ "reasoning_type": "str",+ "reasoning_length": len("No candidates match."),}warning_event = zero_selection_warnings[0]for field, value in expected_fields.items():assert warning_event[field] == value+ assert "No candidates match." not in repr(warning_event)@pytest.mark.asynciodiff --git a/tests/unit/test_handle_terminate_action.py b/tests/unit/test_handle_terminate_action.pyindex b272113f5..527b32b0e 100644--- a/tests/unit/test_handle_terminate_action.py+++ b/tests/unit/test_handle_terminate_action.py@@ -8,6 +8,7 @@ from typing import Anyfrom unittest.mock import AsyncMock, MagicMock, patchimport pytest+from structlog.testing import capture_logsfrom skyvern.errors.errors import TimeoutGetTOTPVerificationCodeError, UserDefinedErrorfrom skyvern.webeye.actions.actions import TerminateAction@@ -182,6 +183,41 @@ async def test_terminate_preserves_errors_when_extract_fails() -> None:assert action.errors[0].error_code == "OTP_TIMEOUT"+async def test_terminate_does_not_log_malformed_llm_error_payload(monkeypatch: pytest.MonkeyPatch) -> None:+ sentinel = "private-malformed-terminate-error-sentinel"+ task = _make_task(error_code_mapping={"LOGIN_FAILED": "the login did not complete"})+ task.navigation_goal = "Log in"+ task.navigation_payload = {}+ action = TerminateAction(reasoning="Page validation failed", errors=[])++ with (+ _surface_errors_llm_returns(+ monkeypatch,+ [{"error_code": sentinel, "reasoning": sentinel, "confidence_float": sentinel}],+ ),+ capture_logs() as events,+ ):+ results = await handle_terminate_action(+ action=action,+ page=MagicMock(),+ scraped_page=_make_scraped_page(),+ task=task,+ step=_make_step(),+ )++ assert len(results) == 1+ assert isinstance(results[0], ActionSuccess)+ event = next(+ event+ for event in events+ if event["event"] == "extract_user_defined_errors failed, using errors from action reasoning"+ )+ assert event["error_type"] == "ValidationError"+ assert "exc_info" not in event+ assert sentinel not in repr(event)++@pytest.mark.asyncioasync def test_extract_user_defined_errors_falls_back_to_reasoning_match(monkeypatch: pytest.MonkeyPatch) -> None:task = _make_task(error_code_mapping={"portal_inaccessible": "portal is inaccessible"})diff --git a/tests/unit/test_llm_response_parsing.py b/tests/unit/test_llm_response_parsing.pyindex 389f5467b..6000d853b 100644--- a/tests/unit/test_llm_response_parsing.py+++ b/tests/unit/test_llm_response_parsing.py@@ -42,6 +42,22 @@ def test_coerce_response_to_dict_variants(response, expected):assert expected[1]+ ("response", "expected"),+ [+ ({"llm_response": "private-sentinel"}, {"response_type": "dict", "response_keys_count": 1}),+ (["private-sentinel"], {"response_type": "list", "response_length": 1}),+ ("private-sentinel", {"response_type": "str", "response_length": 16}),+ (None, {"response_type": "NoneType"}),+ ],+)+def test_llm_response_log_metadata_contains_only_shape(response, expected):+ metadata = llm_utils.llm_response_log_metadata(response)++ assert metadata == expected+ assert "private-sentinel" not in repr(metadata)++def test_coerce_response_warning_keeps_metadata_without_response_content():sentinel = "private-response-sentinel"response = [{"reasoning": sentinel}]@@ -53,6 +69,11 @@ def test_coerce_response_warning_keeps_metadata_without_response_content():assert event["response_type"] == "list"assert event["response_length"] == 1assert event["prompt_name"] == "extract-actions"+ first_item_event = next(+ event for event in events if event["event"] == "Parsed LLM response is a list; using first dict element"+ )+ assert first_item_event["first_item_keys_count"] == 1+ assert "first_item_keys" not in first_item_eventassert all(sentinel not in repr(event) for event in events)diff --git a/tests/unit/test_task_v2_summary_envelope.py b/tests/unit/test_task_v2_summary_envelope.pyindex 52c962575..fdfb9aac4 100644--- a/tests/unit/test_task_v2_summary_envelope.py+++ b/tests/unit/test_task_v2_summary_envelope.py@@ -13,6 +13,7 @@ from datetime import timezonefrom types import SimpleNamespaceimport pytest+from structlog.testing import capture_logsfrom skyvern.forge.prompts import prompt_enginefrom skyvern.services import task_v2_service@@ -67,6 +68,20 @@ def test_parser_recovers_top_level_payload() -> None:assert output == _NON_ENVELOPE_RESPONSE+def test_parser_recovery_logs_only_key_count() -> None:+ response = {"private-model-key": "private-model-value"}++ with capture_logs() as events:+ assert _parse_task_v2_summary_response(response) == (None, response)++ event = next(+ event for event in events if event["event"] == "Recovered task_v2 summary output from non-envelope response"+ )+ assert event["recovered_keys_count"] == 1+ assert "private-model-key" not in repr(event)+ assert "private-model-value" not in repr(event)++def test_parser_recovery_preserves_user_schema_description_key() -> None:# A goal-defined contract may itself use "description"; recovery must not strip it.resp = {"description": "Widget", "price": 10}diff --git a/tests/unit/test_text_prompt_block.py b/tests/unit/test_text_prompt_block.pyindex 9bbc32fdd..2926d21a0 100644--- a/tests/unit/test_text_prompt_block.py+++ b/tests/unit/test_text_prompt_block.py@@ -864,6 +864,26 @@ def _patch_send_prompt_deps(monkeypatch, *, handler) -> None:monkeypatch.setattr(block_module.asyncio, "sleep", AsyncMock())+async def test_text_prompt_response_log_omits_llm_output(monkeypatch):+ from structlog.testing import capture_logs++ sentinel = "private-llm-response-sentinel"++ async def fake_llm(**kwargs):+ return {"llm_response": sentinel}++ _patch_send_prompt_deps(monkeypatch, handler=fake_llm)+ block = _make_text_prompt_block(prompt="Return a synthetic response.")++ with capture_logs() as events:+ result = await block.send_prompt(block.prompt, workflow_run_id="wr_qa", organization_id="o_qa")++ assert result == {"llm_response": sentinel}+ response_log = next(event for event in events if event["event"] == "TextPromptBlock Received response from LLM")+ assert sentinel not in repr(response_log)++@pytest.mark.asyncioasync def test_text_prompt_block_retries_transient_llm_provider_error(monkeypatch):"""A transient provider error retries with backoff and recovers instead of failing."""