Skip to content

Automate browser based workflows with AI

Document round-one self-hosted identity QA results (#9060)

Aaron Perez4bf9d1a5bedb
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.py
index 63b7baec4..3e9f41951 100644
--- a/skyvern/core/script_generations/real_skyvern_page_ai.py
+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py
@@ -23,6 +23,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import (
get_org_aware_secondary_llm_api_handler,
)
from skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_result
+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata
from skyvern.forge.sdk.cache import extraction_cache
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_prompt_type_handler
@@ -887,9 +888,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
LOG.info(
"page.classify: LLM classification result",
- classification=classification,
- confidence=confidence,
- reasoning=reasoning,
+ **llm_response_log_metadata(json_response),
)
if classification in options or classification == "UNKNOWN":
@@ -1660,7 +1659,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
LOG.warning("ai_extract cache store failed; ignoring", exc_info=True)
if context and context.script_mode:
- LOG.debug("Extracted information", result=result)
+ LOG.debug("Extracted information", **llm_response_log_metadata(result))
return result
async def ai_validate(
@@ -1720,14 +1719,12 @@ class RealSkyvernPageAi(SkyvernPageAi):
if not result or not isinstance(result, dict):
LOG.error(
"AI locate element failed - invalid result",
- result=result,
- result_type=type(result).__name__,
+ **llm_response_log_metadata(result),
prompt=prompt_rendered,
)
return None
element_id = result.get("element_id", None)
- confidence = result.get("confidence_float", 0.0)
xpath: str | None = None
if element_id:
@@ -1741,17 +1738,14 @@ class RealSkyvernPageAi(SkyvernPageAi):
if not xpath:
LOG.error(
"AI locate element failed - no xpath in element data",
- element_id=element_id,
- result=result,
+ **llm_response_log_metadata(result),
prompt=prompt_rendered,
)
return None
LOG.info(
"AI locate element result",
- element_id=element_id,
- xpath=xpath,
- confidence=confidence,
+ **llm_response_log_metadata(result),
prompt=prompt_rendered,
)
@@ -1807,25 +1801,29 @@ class RealSkyvernPageAi(SkyvernPageAi):
)
if not json_response or "inferred_actions" not in json_response:
- LOG.warning("ai_act: failed to infer action type", prompt=prompt, response=json_response)
+ LOG.warning(
+ "ai_act: failed to infer action type",
+ prompt=prompt,
+ **llm_response_log_metadata(json_response),
+ )
return
inferred_actions = json_response.get("inferred_actions", [])
if not inferred_actions:
- error = json_response.get("error")
- LOG.warning("ai_act: no action type inferred", prompt=prompt, error=error)
+ LOG.warning(
+ "ai_act: no action type inferred",
+ prompt=prompt,
+ **llm_response_log_metadata(json_response),
+ )
return
action_info = inferred_actions[0]
action_type = action_info.get("action_type")
- confidence = action_info.get("confidence_float", 0.0)
LOG.info(
"ai_act: inferred action type",
prompt=prompt,
- action_type=action_type,
- confidence=confidence,
- reasoning=action_info.get("reasoning"),
+ **llm_response_log_metadata(json_response),
)
if not skip_refresh:
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index b3a89e1a8..e2bb4687c 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -131,7 +131,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (
LLMResponseMissingActionsError,
)
from skyvern.forge.sdk.api.llm.ui_tars_llm_caller import UITarsLLMCaller
-from skyvern.forge.sdk.api.llm.utils import uses_only_anthropic_models
+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata, uses_only_anthropic_models
from skyvern.forge.sdk.api.llm.vertex_cache_manager import get_cache_manager
from skyvern.forge.sdk.api.llm.yutori_navigator_llm_caller import (
YutoriNavigatorLLMCaller,
@@ -6933,7 +6933,7 @@ class ForgeAgent:
screenshots=scraped_page.screenshots,
system_prompt=task.workflow_system_prompt,
)
- LOG.info("Skyvern response to CUA question", skyvern_response=skyvern_response)
+ LOG.info("Skyvern response to CUA question", **llm_response_log_metadata(skyvern_response))
cua_question = cua_question or skyvern_response.get("question_or_decision")
resp_content = skyvern_response.get("answer")
if not resp_content:
@@ -7163,7 +7163,7 @@ class ForgeAgent:
# Generate response using the LLMCaller
response_content = await llm_caller.generate_ui_tars_response(step)
- LOG.info(f"UI-TARS raw response: {response_content}")
+ LOG.info("UI-TARS raw response", **llm_response_log_metadata(response_content))
window_dimension = (
cast(Resolution, scraped_page.window_dimension)
@@ -11116,8 +11116,9 @@ class ForgeAgent:
"Parallel verification: termination required, marking task as terminated",
step_id=step.step_id,
task_id=task.task_id,
- reasoning=skyvern_context.mask_multi_field_totp_data(
- complete_action.reasoning, task_id=task.task_id
+ reasoning_type=type(complete_action.reasoning).__name__,
+ reasoning_length=(
+ len(complete_action.reasoning) if isinstance(complete_action.reasoning, str) else None
),
)
final_status = step.speculative_original_status or StepStatus.completed
@@ -11496,8 +11497,8 @@ class ForgeAgent:
"Dropped LLM-returned error codes not in user error_code_mapping",
task_id=task.task_id,
step_id=step.step_id,
- dropped_codes=dropped,
- allowed_codes=sorted((task.error_code_mapping or {}).keys()),
+ dropped_count=len(dropped),
+ allowed_codes_count=len(task.error_code_mapping or {}),
)
return kept
@@ -11985,7 +11986,7 @@ class ForgeAgent:
"Step completed and goal achieved, marking task as completed",
step_order=step.order,
step_retry=step.retry_index,
- output=step.output,
+ **llm_response_log_metadata(step.output.model_dump() if step.output is not None else None),
)
last_step = await self.update_step(step, is_last=True)
extracted_information = await self.get_extracted_information_for_task(task)
@@ -12036,9 +12037,7 @@ class ForgeAgent:
"Step completed and terminated by the agent, marking task as terminated",
step_order=step.order,
step_retry=step.retry_index,
- output=skyvern_context.mask_multi_field_totp_data(step.output.model_dump(), task_id=task.task_id)
- if step.output is not None and task.task_id in skyvern_context.multi_field_totp_masking_task_ids()
- else step.output,
+ **llm_response_log_metadata(step.output.model_dump() if step.output is not None else None),
)
last_step = await self.update_step(step, is_last=True)
failure_category = classify_from_failure_reason(failure_reason)
@@ -12076,7 +12075,7 @@ class ForgeAgent:
"Step completed for the action block, marking task as completed",
step_order=step.order,
step_retry=step.retry_index,
- output=step.output,
+ **llm_response_log_metadata(step.output.model_dump() if step.output is not None else None),
)
last_step = await self.update_step(step, is_last=True)
await self.update_task(
diff --git a/skyvern/forge/sdk/api/llm/utils.py b/skyvern/forge/sdk/api/llm/utils.py
index d61844275..63f98321d 100644
--- a/skyvern/forge/sdk/api/llm/utils.py
+++ b/skyvern/forge/sdk/api/llm/utils.py
@@ -277,6 +277,16 @@ def _reassemble_list_response(response: list[Any], allow_action_list_wrap: bool)
return None
+def llm_response_log_metadata(response: Any) -> dict[str, str | int]:
+ """Return content-free metadata suitable for logging an LLM response."""
+ metadata: dict[str, str | int] = {"response_type": type(response).__name__}
+ if isinstance(response, dict):
+ metadata["response_keys_count"] = len(response)
+ elif isinstance(response, (list, str)):
+ metadata["response_length"] = len(response)
+ return metadata
+
+
def _coerce_response_to_dict(response: Any, prompt_name: str | None = None) -> dict[str, Any]:
"""Ensure parsed LLM responses expose a dict interface to callers."""
if isinstance(response, dict):
@@ -295,7 +305,7 @@ def _coerce_response_to_dict(response: Any, prompt_name: str | None = None) -> d
LOG.info(
"Recovered single-object response from list-shaped LLM output",
response_length=len(response),
- recovered_keys=list(reassembled.keys()),
+ recovered_keys_count=len(reassembled),
prompt_name=prompt_name,
)
return reassembled
@@ -305,7 +315,7 @@ def _coerce_response_to_dict(response: Any, prompt_name: str | None = None) -> d
"Parsed LLM response is a list; using first dict element",
response_length=len(response),
first_item_type=type(response[0]).__name__ if response else None,
- first_item_keys=list(first_dict.keys()) if first_dict else None,
+ first_item_keys_count=len(first_dict) if first_dict else None,
prompt_name=prompt_name,
)
if first_dict is not None:
diff --git a/skyvern/forge/sdk/routes/prompts.py b/skyvern/forge/sdk/routes/prompts.py
index 72535bd0a..5f9e2b6f0 100644
--- a/skyvern/forge/sdk/routes/prompts.py
+++ b/skyvern/forge/sdk/routes/prompts.py
@@ -13,6 +13,7 @@ from skyvern.forge.sdk.api.llm.exceptions import (
InvalidLLMResponseType,
LLMProviderError,
)
+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata
from skyvern.forge.sdk.routes.routers import base_router
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.prompts import (
@@ -119,7 +120,7 @@ async def improve_prompt(
use_case=use_case,
organization_id=current_org.organization_id,
prompt=request.prompt,
- improved_prompt=output,
+ **llm_response_log_metadata(output),
)
response = ImprovePromptResponse(
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index ff42f0a59..70473aaaf 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -167,6 +167,7 @@ from skyvern.forge.sdk.api.llm.schema_validator import (
resolve_schema_type,
validate_schema,
)
+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.artifact.storage.base import get_download_retry_started_at, is_file_from_retry_attempt
from skyvern.forge.sdk.copilot.code_block_security import SlottedReference, is_inert_slot, slot_template
@@ -2981,7 +2982,7 @@ class BaseTaskBlock(Block):
organization_id=workflow_run.organization_id,
current_retry=current_retry,
max_retries=self.max_retries,
- task_output=task_output.model_dump_json(),
+ **llm_response_log_metadata(task_output.model_dump()),
)
if not will_retry:
output_parameter_value = task_output.model_dump()
@@ -11297,7 +11298,7 @@ class TextPromptBlock(Block):
except Exception as e:
LOG.error("Failed to save TextPromptBlock artifacts", error=e)
- LOG.info("TextPromptBlock Received response from LLM", response=response)
+ LOG.info("TextPromptBlock Received response from LLM", **llm_response_log_metadata(response))
return response
async def send_prompt_with_schema_retries(
@@ -12870,7 +12871,8 @@ class FileUploadBlock(FileDestinationBlock):
workflow_run_block_id=workflow_run_block_id,
candidate_count=candidate_count,
selected_count=selected_count,
- reasoning=selection_reasoning,
+ reasoning_type=type(selection_reasoning).__name__,
+ reasoning_length=len(selection_reasoning) if isinstance(selection_reasoning, str) else None,
)
await self.record_output_parameter_value(workflow_run_context, workflow_run_id, uploaded_uris)
return await self.build_block_result(
@@ -16697,7 +16699,8 @@ class FileDownloadBlock(BaseTaskBlock, FileDestinationBlock):
workflow_run_block_id=workflow_run_block_id,
candidate_count=candidate_count,
selected_count=selected_count,
- reasoning=selection_reasoning,
+ reasoning_type=type(selection_reasoning).__name__,
+ reasoning_length=len(selection_reasoning) if isinstance(selection_reasoning, str) else None,
)
return result
@@ -18714,7 +18717,8 @@ def _parse_single_evaluation(
LOG.warning(
"Conditional branch evaluation returned non-boolean result",
branch_index=idx,
- result=result,
+ result_type=type(result).__name__,
+ result_length=len(result) if isinstance(result, str) else None,
evaluated_result=bool_result,
)
@@ -19273,8 +19277,8 @@ async def _evaluate_prompt_branch_conditions_batch(
block_label=log_label,
attempt=attempt,
will_retry=attempt + 1 < MAX_PROMPT_BRANCH_EVAL_ATTEMPTS,
- error=str(malformed),
- raw_output=raw_output_value,
+ error_type=type(malformed).__name__,
+ **llm_response_log_metadata(raw_output_value),
)
continue
@@ -19298,10 +19302,10 @@ async def _evaluate_prompt_branch_conditions_batch(
LOG.info(
"Conditional branch evaluation results",
block_label=log_label,
- results=results_array,
- llm_rendered_expressions=llm_rendered_expressions,
+ results_count=len(results_array),
+ llm_rendered_expressions_count=len(llm_rendered_expressions),
attempt=attempt,
- raw_output=normalized_output,
+ **llm_response_log_metadata(normalized_output),
)
return (results_array, llm_rendered_expressions, extraction_goal, normalized_output)
@@ -19309,7 +19313,7 @@ async def _evaluate_prompt_branch_conditions_batch(
"Conditional branch evaluation failed after retries",
block_label=log_label,
attempts=MAX_PROMPT_BRANCH_EVAL_ATTEMPTS,
- error=str(last_malformed),
+ error_type=type(last_malformed).__name__ if last_malformed is not None else None,
)
raise ConditionalBranchEvaluationError(f"Conditional branch evaluation failed: {last_malformed}")
diff --git a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
index 7b9ae5827..40ddeb0c1 100644
--- a/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
+++ b/skyvern/forge/sdk/workflow/models/pdf_fill_block.py
@@ -458,7 +458,7 @@ class PdfFillBlock(Block):
LOG.warning(
"PdfFillBlock LLM response missing expected key",
expected_key=expected_key,
- response_keys=list(parsed.keys()),
+ response_keys_count=len(parsed),
)
return parsed
diff --git a/skyvern/forge/sdk/workflow/models/split_pdf_block.py b/skyvern/forge/sdk/workflow/models/split_pdf_block.py
index 1cab81a11..36d1d2959 100644
--- a/skyvern/forge/sdk/workflow/models/split_pdf_block.py
+++ b/skyvern/forge/sdk/workflow/models/split_pdf_block.py
@@ -171,7 +171,7 @@ class SplitPdfBlock(Block):
LOG.warning(
"SplitPdfBlock LLM response missing expected key",
expected_key=expected_key,
- response_keys=list(parsed.keys()),
+ response_keys_count=len(parsed),
)
return parsed
diff --git a/skyvern/services/error_detection_service.py b/skyvern/services/error_detection_service.py
index 47beb14ba..d1d583d28 100644
--- a/skyvern/services/error_detection_service.py
+++ b/skyvern/services/error_detection_service.py
@@ -17,6 +17,7 @@ from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_primary_llm_api_handler
+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.tasks import Task
@@ -156,7 +157,6 @@ async def _detect_errors_from_page(
task_id=task.task_id,
step_id=step.step_id,
error_count=len(user_defined_errors),
- errors=[e.error_code for e in user_defined_errors],
)
return user_defined_errors
@@ -238,13 +238,13 @@ async def _detect_errors_from_context(
try:
user_defined_error = UserDefinedError.model_validate(error_dict)
user_defined_errors.append(user_defined_error)
- except Exception:
+ except Exception as exc:
LOG.warning(
"Failed to validate user-defined error",
task_id=task.task_id,
step_id=step.step_id,
- error_dict=error_dict,
- exc_info=True,
+ **llm_response_log_metadata(error_dict),
+ error_type=type(exc).__name__,
)
user_defined_errors, dropped = filter_to_user_defined_codes(user_defined_errors, task.error_code_mapping)
@@ -253,8 +253,8 @@ async def _detect_errors_from_context(
"Dropped LLM-returned error codes not in user error_code_mapping",
task_id=task.task_id,
step_id=step.step_id,
- dropped_codes=dropped,
- allowed_codes=sorted((task.error_code_mapping or {}).keys()),
+ dropped_count=len(dropped),
+ allowed_codes_count=len(task.error_code_mapping or {}),
)
LOG.info(
@@ -262,16 +262,15 @@ async def _detect_errors_from_context(
task_id=task.task_id,
step_id=step.step_id,
error_count=len(user_defined_errors),
- errors=[e.error_code for e in user_defined_errors],
)
return user_defined_errors
- except Exception:
+ except Exception as exc:
LOG.warning(
"Failed to detect errors from context",
task_id=task.task_id,
step_id=step.step_id,
- exc_info=True,
+ error_type=type(exc).__name__,
)
return []
diff --git a/skyvern/services/pdf_import_service.py b/skyvern/services/pdf_import_service.py
index e56b7b415..be03e230a 100644
--- a/skyvern/services/pdf_import_service.py
+++ b/skyvern/services/pdf_import_service.py
@@ -9,6 +9,7 @@ from fastapi import HTTPException
from skyvern.config import settings
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.utils.pdf_parser import extract_pdf_file
from skyvern.schemas.workflows import WorkflowCreateYAMLRequest
@@ -215,8 +216,7 @@ class PDFImportService:
LOG.info(
"LLM response received",
- response_type=type(response),
- response_keys=list(response.keys()) if isinstance(response, dict) else None,
+ **llm_response_log_metadata(response),
organization_id=organization.organization_id,
)
@@ -225,8 +225,7 @@ class PDFImportService:
if not isinstance(response, dict):
LOG.error(
"LLM returned non-dict response",
- response_type=type(response),
- response=str(response)[:500],
+ **llm_response_log_metadata(response),
organization_id=organization.organization_id,
)
raise HTTPException(status_code=422, detail="LLM returned invalid response format - expected JSON object")
@@ -235,7 +234,7 @@ class PDFImportService:
if response.get("error") == "not_sop":
LOG.info(
"LLM detected non-SOP content",
- reason=response.get("reason"),
+ **llm_response_log_metadata(response),
organization_id=organization.organization_id,
)
raise HTTPException(
@@ -249,7 +248,7 @@ class PDFImportService:
if "workflow_definition" not in response:
LOG.error(
"LLM response missing workflow_definition",
- response_keys=list(response.keys()),
+ **llm_response_log_metadata(response),
organization_id=organization.organization_id,
)
raise HTTPException(status_code=422, detail="LLM response missing 'workflow_definition' field")
@@ -257,7 +256,7 @@ class PDFImportService:
if "blocks" not in response.get("workflow_definition", {}):
LOG.error(
"LLM workflow_definition missing blocks",
- workflow_def_keys=list(response.get("workflow_definition", {}).keys()),
+ workflow_def_keys_count=len(response.get("workflow_definition", {})),
organization_id=organization.organization_id,
)
raise HTTPException(status_code=422, detail="LLM workflow definition missing 'blocks' field")
@@ -269,17 +268,15 @@ class PDFImportService:
LOG.info(
"Workflow JSON validated successfully",
- title=response.get("title"),
+ **llm_response_log_metadata(response),
block_count=len(response.get("workflow_definition", {}).get("blocks", [])),
organization_id=organization.organization_id,
)
except Exception as e:
LOG.error(
"Failed to validate workflow request",
- error=str(e),
error_type=type(e).__name__,
organization_id=organization.organization_id,
- exc_info=True,
)
raise HTTPException(
status_code=422,
diff --git a/skyvern/services/script_reviewer.py b/skyvern/services/script_reviewer.py
index f849f927b..974e31fe5 100644
--- a/skyvern/services/script_reviewer.py
+++ b/skyvern/services/script_reviewer.py
@@ -26,6 +26,7 @@ from skyvern.core.script_generations.script_validators import (
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_secondary_llm_api_handler
+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata
from skyvern.forge.sdk.workflow.models.block import get_all_blocks
from skyvern.forge.sdk.workflow.models.parameter import is_sensitive_workflow_parameter
from skyvern.schemas.scripts import ScriptBranchHit, ScriptFallbackEpisode
@@ -1020,7 +1021,7 @@ class ScriptReviewer:
"ScriptReviewer: triage result",
episode_id=episode.episode_id,
code_fixable=is_code_fixable,
- triage_response=response_text[:200] if response_text else None,
+ **llm_response_log_metadata(response_text),
)
return is_code_fixable
except Exception:
@@ -1222,8 +1223,7 @@ class ScriptReviewer:
block_label=block_label,
attempt=attempt,
prompt_name=template,
- response_type=type(llm_response).__name__,
- response_snippet=str(llm_response)[:200],
+ **llm_response_log_metadata(llm_response),
)
updated_code = self._extract_code_from_response(
@@ -1237,7 +1237,7 @@ class ScriptReviewer:
block_label=block_label,
attempt=attempt,
prompt_name=template,
- response_snippet=str(llm_response)[:500],
+ **llm_response_log_metadata(llm_response),
)
if attempt >= max_attempts:
return None
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 15237afc2..97b7e81dc 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -56,6 +56,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import (
get_org_aware_primary_llm_api_handler,
get_org_aware_secondary_llm_api_handler,
)
+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.artifact.storage.base import get_download_retry_started_at, is_file_from_retry_attempt
from skyvern.forge.sdk.core import skyvern_context
@@ -1498,7 +1499,7 @@ async def _detect_user_defined_errors(
except Exception:
LOG.warning(
"Failed to validate user-defined error",
- error_dict=error_dict,
+ **llm_response_log_metadata(error_dict),
)
user_defined_errors, dropped = filter_to_user_defined_codes(user_defined_errors, error_code_mapping)
@@ -1507,8 +1508,8 @@ async def _detect_user_defined_errors(
"Dropped LLM-returned error codes not in user error_code_mapping",
task_id=task.task_id,
step_id=step.step_id,
- dropped_codes=dropped,
- allowed_codes=sorted((error_code_mapping or {}).keys()),
+ dropped_count=len(dropped),
+ allowed_codes_count=len(error_code_mapping or {}),
)
LOG.info(
@@ -1516,7 +1517,6 @@ async def _detect_user_defined_errors(
task_id=task.task_id,
step_id=step.step_id,
error_count=len(user_defined_errors),
- errors=[e.error_code for e in user_defined_errors],
)
return user_defined_errors
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index aad05eb1f..a6fc494e1 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -37,6 +37,7 @@ from skyvern.forge.sdk.api.llm.custom_llm_registry import (
ensure_custom_llm_model_registered_for_org,
)
from skyvern.forge.sdk.api.llm.exceptions import EmptyLLMResponseError, InvalidLLMResponseFormat
+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.hashing import generate_url_hash
@@ -513,7 +514,7 @@ async def initialize_task_v2_metadata(
)
# validate
- LOG.info(f"Initialized task v2 initial response: {metadata_response}")
+ LOG.info("Initialized task v2 initial response", **llm_response_log_metadata(metadata_response))
url: str = user_url or metadata_response.get("url", "")
if not url:
raise UrlGenerationFailure()
@@ -1086,7 +1087,7 @@ async def run_task_v2_helper(
)
LOG.info(
"Task v2 response",
- task_v2_response=task_v2_response,
+ **llm_response_log_metadata(task_v2_response),
iteration=i,
current_url=current_url,
workflow_run_id=workflow_run_id,
@@ -1161,12 +1162,12 @@ async def run_task_v2_helper(
return workflow, workflow_run, task_v2
if not plan:
- LOG.warning("No plan found in task v2 response", task_v2_response=task_v2_response)
+ LOG.warning("No plan found in task v2 response", **llm_response_log_metadata(task_v2_response))
continue
# parse task v2 response and run the next task
if not task_type:
- LOG.error("No task type found in task v2 response", task_v2_response=task_v2_response)
+ LOG.error("No task type found in task v2 response", **llm_response_log_metadata(task_v2_response))
task_v2 = await mark_task_v2_as_failed(
task_v2_id=task_v2_id,
workflow_run_id=workflow_run_id,
@@ -1424,10 +1425,10 @@ async def run_task_v2_helper(
)
LOG.info(
"Task v2 completion check response",
- completion_resp=completion_resp,
+ **llm_response_log_metadata(completion_resp),
iteration=i,
workflow_run_id=workflow_run_id,
- task_history=task_history,
+ task_history_count=len(task_history),
)
user_goal_achieved = completion_resp.get("user_goal_achieved", False)
should_terminate = completion_resp.get("should_terminate", False)
@@ -1449,7 +1450,7 @@ async def run_task_v2_helper(
"User goal achieved according to the task v2 completion check",
iteration=i,
workflow_run_id=workflow_run_id,
- completion_resp=completion_resp,
+ **llm_response_log_metadata(completion_resp),
)
lost_task_v2 = await _fail_task_v2_after_browser_loss(
task_v2_id, workflow_run_id, organization_id, completed_block=(block, block_result)
@@ -1901,7 +1902,7 @@ async def _generate_loop_task(
prompt_name="task_v2_generate_task_block",
system_prompt=task_v2.workflow_system_prompt,
)
- LOG.info("Task in loop metadata response", task_in_loop_metadata_response=task_in_loop_metadata_response)
+ LOG.info("Task in loop metadata response", **llm_response_log_metadata(task_in_loop_metadata_response))
navigation_goal = task_in_loop_metadata_response.get("navigation_goal")
data_extraction_goal = task_in_loop_metadata_response.get("data_extraction_goal")
data_extraction_schema = task_in_loop_metadata_response.get("data_schema")
@@ -2021,7 +2022,7 @@ async def _generate_extraction_task(
raise last_exc # type: ignore[misc]
assert generate_extraction_task_response is not None
- LOG.info("Data extraction response", data_extraction_response=generate_extraction_task_response)
+ LOG.info("Data extraction response", **llm_response_log_metadata(generate_extraction_task_response))
# create OutputParameter for the data_extraction block
data_schema: dict[str, Any] | list | None = generate_extraction_task_response.get("schema")
@@ -2134,7 +2135,11 @@ async def _generate_compute_task(
plan: str,
task_history: list[dict] | None = None,
) -> tuple[CodeBlock, list[BLOCK_YAML_TYPES], list[PARAMETER_YAML_TYPES]]:
- LOG.info("Generating compute task", plan=plan)
+ LOG.info(
+ "Generating compute task",
+ plan_type=type(plan).__name__,
+ plan_length=len(plan) if isinstance(plan, str) else None,
+ )
context = skyvern_context.ensure_context()
gathered = [record for record in (task_history or []) if record.get("extracted_data") is not None]
if not gathered:
@@ -2647,7 +2652,7 @@ def _get_extracted_data_from_block_result(
if not isinstance(inner_loop_output, list):
LOG.warning(
"Inner loop output is not a list",
- inner_loop_output=inner_loop_output,
+ **llm_response_log_metadata(inner_loop_output),
task_v2_id=task_v2_id,
workflow_run_id=workflow_run_id,
workflow_run_block_id=block_result.workflow_run_block_id,
@@ -2657,7 +2662,7 @@ def _get_extracted_data_from_block_result(
if not isinstance(inner_output, dict):
LOG.warning(
"inner output is not a dict",
- inner_output=inner_output,
+ **llm_response_log_metadata(inner_output),
task_v2_id=task_v2_id,
workflow_run_id=workflow_run_id,
workflow_run_block_id=block_result.workflow_run_block_id,
@@ -2667,7 +2672,7 @@ def _get_extracted_data_from_block_result(
if not isinstance(output_value, dict):
LOG.warning(
"output_value is not a dict",
- output_value=output_value,
+ **llm_response_log_metadata(output_value),
task_v2_id=task_v2_id,
workflow_run_id=workflow_run_id,
workflow_run_block_id=block_result.workflow_run_block_id,
@@ -2814,7 +2819,7 @@ def _parse_task_v2_summary_response(resp: Any) -> tuple[str | None, Any]:
}
LOG.info(
"Recovered task_v2 summary output from non-envelope response",
- recovered_keys=sorted(output.keys()),
+ recovered_keys_count=len(output),
)
return description, output
@@ -2851,7 +2856,11 @@ async def _generate_task_v2_deliverable(
prompt_name="task_v2_summary",
system_prompt=task_v2.workflow_system_prompt,
)
- LOG.info("Task v2 summary response", task_v2_summary_resp=task_v2_summary_resp, is_partial=is_partial)
+ LOG.info(
+ "Task v2 summary response",
+ **llm_response_log_metadata(task_v2_summary_resp),
+ is_partial=is_partial,
+ )
summary_description, summarized_output = _parse_task_v2_summary_response(task_v2_summary_resp)
await app.DATABASE.observer.update_thought(
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index e511c1dbd..2dea4e2f1 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -130,6 +130,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import (
)
from skyvern.forge.sdk.api.llm.exceptions import BaseLLMError, LLMProviderError
from skyvern.forge.sdk.api.llm.schema_validator import extraction_shape_matches, validate_and_fill_extraction_result
+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata
from skyvern.forge.sdk.artifact.storage.base import get_download_retry_started_at, is_file_from_retry_attempt
from skyvern.forge.sdk.browser_action_preflight import preflight_action, preflight_derived_action
from skyvern.forge.sdk.cache import extraction_cache, extraction_shadow
@@ -11513,13 +11514,13 @@ async def handle_terminate_action(
extracted_errors = await extract_user_defined_errors(
task=task, step=step, scraped_page=scraped_page, reasoning=action.reasoning
)
- except Exception:
+ except Exception as exc:
LOG.warning(
"extract_user_defined_errors failed, using errors from action reasoning",
task_id=task.task_id,
step_id=step.step_id,
- action_errors=action.errors,
- exc_info=True,
+ action_error_count=len(action.errors),
+ error_type=type(exc).__name__,
)
else:
skyvern_codes = {error.error_code for error in skyvern_errors}
@@ -13466,7 +13467,7 @@ async def input_or_auto_complete_input(
if input_or_select_context.is_search_bar:
LOG.info(
"Stop generating potential values for the auto-completion since it's a search bar",
- context=input_or_select_context,
+ **llm_response_log_metadata(input_or_select_context.model_dump()),
)
return None
@@ -13620,7 +13621,7 @@ async def input_or_auto_complete_input(
LOG.info(
"Auto completion didn't finish, this might leave the input value to be empty.",
sampling=True,
- context=input_or_select_context,
+ **llm_response_log_metadata(input_or_select_context.model_dump()),
attempts=attempt_timings,
total_seconds=round(_elapsed_seconds(), 1),
)
@@ -13845,7 +13846,7 @@ async def sequentially_select_from_dropdown(
if not force_select and input_or_select_context.is_search_bar:
LOG.info(
"Exit custom selection mode since it's a non-force search bar",
- context=input_or_select_context,
+ **llm_response_log_metadata(input_or_select_context.model_dump()),
)
return None
@@ -15422,8 +15423,7 @@ async def select_from_emerging_elements(
LOG.info(
"LLM response for the matched element",
sampling=True,
- matched_value=value,
- response=json_response,
+ **llm_response_log_metadata(json_response),
)
# Check the no-match shape before ``ActionType`` coercion — coercing an empty
@@ -15861,8 +15861,7 @@ async def select_from_dropdown(
LOG.info(
"LLM response for the matched element",
sampling=True,
- matched_value=value,
- response=json_response,
+ **llm_response_log_metadata(json_response),
)
# Check the no-match shape before ``ActionType`` coercion — coercing an empty
@@ -16456,7 +16455,7 @@ async def normal_select(
)
LOG.debug(
"Parsed input/select context",
- context=input_or_select_context,
+ **llm_response_log_metadata(input_or_select_context.model_dump()),
)
select_options_result = await skyvern_element.refresh_select_options()
@@ -17300,7 +17299,7 @@ async def _get_input_or_select_context(
original_response_type=type(json_response).__name__,
original_response_length=len(json_response) if json_response else 0,
first_item_type=type(json_response[0]).__name__ if json_response else None,
- first_item_keys=list(json_response[0].keys())
+ first_item_keys_count=len(json_response[0])
if json_response and isinstance(json_response[0], dict)
else None,
)
@@ -17310,7 +17309,7 @@ async def _get_input_or_select_context(
input_or_select_context = InputOrSelectContext.model_validate(json_response)
LOG.debug(
"Parsed input/select context",
- context=input_or_select_context,
+ **llm_response_log_metadata(input_or_select_context.model_dump()),
)
return input_or_select_context
@@ -17385,8 +17384,8 @@ async def extract_user_defined_errors(
"Dropped LLM-returned error codes not in user error_code_mapping",
task_id=task.task_id,
step_id=step.step_id,
- dropped_codes=dropped,
- allowed_codes=sorted((task.error_code_mapping or {}).keys()),
+ dropped_count=len(dropped),
+ allowed_codes_count=len(task.error_code_mapping or {}),
)
if not kept and reasoning:
return _match_user_defined_error_from_reasoning(task=task, step=step, reasoning=reasoning)
diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.py
index 5fcfe7e26..4b54f2218 100644
--- a/skyvern/webeye/actions/parse_actions.py
+++ b/skyvern/webeye/actions/parse_actions.py
@@ -19,6 +19,7 @@ from skyvern.exceptions import (
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.llm.api_handler_factory import get_org_aware_primary_llm_api_handler
+from skyvern.forge.sdk.api.llm.utils import llm_response_log_metadata
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.tasks import Task
@@ -247,7 +248,10 @@ def parse_action(
if key:
is_single_char = len(key) == 1 and (key.isdigit() or key.isalpha())
if key not in allowed_keys and not is_single_char:
- LOG.warning("KEYPRESS action has unsupported key, skipping action", key=key)
+ LOG.warning(
+ "KEYPRESS action has unsupported key, skipping action",
+ **llm_response_log_metadata(key),
+ )
return NullAction(**base_action_dict)
keys = [key]
else:
@@ -262,7 +266,10 @@ def parse_action(
base_action_dict["skyvern_element_data"] = None
direction = action.get("direction", "down").lower()
if direction not in ("up", "down"):
- LOG.warning("SCROLL action has unexpected direction, defaulting to down", direction=direction)
+ LOG.warning(
+ "SCROLL action has unexpected direction, defaulting to down",
+ **llm_response_log_metadata(direction),
+ )
direction = "down"
if direction == "up":
scroll_x = 0
@@ -281,7 +288,7 @@ def parse_action(
except (TypeError, ValueError):
LOG.warning(
"CLOSE_PAGE action returned with a non-integer tab_index, closing the current tab instead",
- raw_action=action,
+ **llm_response_log_metadata(action),
)
return ClosePageAction(**base_action_dict)
return ClosePageAction(**base_action_dict, tab_index=tab_index)
@@ -291,7 +298,10 @@ def parse_action(
base_action_dict["skyvern_element_hash"] = None
base_action_dict["skyvern_element_data"] = None
if not data_extraction_goal:
- LOG.warning("EXTRACT action returned without a data extraction goal, skipping action", raw_action=action)
+ LOG.warning(
+ "EXTRACT action returned without a data extraction goal, skipping action",
+ **llm_response_log_metadata(action),
+ )
return NullAction(**base_action_dict)
return ExtractAction(
**base_action_dict,
@@ -305,18 +315,27 @@ def parse_action(
base_action_dict["skyvern_element_data"] = None
url = action.get("url")
if not url or not isinstance(url, str):
- LOG.warning("GOTO_URL action returned without a url, skipping action", raw_action=action)
+ LOG.warning(
+ "GOTO_URL action returned without a url, skipping action",
+ **llm_response_log_metadata(action),
+ )
return NullAction(**base_action_dict)
try:
validated_url = prepend_scheme_and_validate_url(url.strip())
except InvalidUrl:
- LOG.warning("GOTO_URL action returned with an invalid url, skipping action", url=url)
+ LOG.warning(
+ "GOTO_URL action returned with an invalid url, skipping action",
+ **llm_response_log_metadata(url),
+ )
return NullAction(**base_action_dict)
# LLM-generated navigation targets can be steered by page content, so block
# private/loopback/link-local hosts the same way the task URL boundary does.
host = urlparse(validated_url).hostname
if not host or is_blocked_host(host, resolve_dns=False):
- LOG.warning("GOTO_URL action targets a blocked host, skipping action", url=validated_url)
+ LOG.warning(
+ "GOTO_URL action targets a blocked host, skipping action",
+ **llm_response_log_metadata(validated_url),
+ )
return NullAction(**base_action_dict)
return GotoUrlAction(**base_action_dict, url=validated_url)
@@ -328,28 +347,43 @@ def parse_action(
if action_type == ActionType.NEW_TAB:
url = action.get("url")
if not url or not isinstance(url, str):
- LOG.warning("NEW_TAB action returned without a url, skipping action", raw_action=action)
+ LOG.warning(
+ "NEW_TAB action returned without a url, skipping action",
+ **llm_response_log_metadata(action),
+ )
return NullAction(**base_action_dict)
try:
validated_url = prepend_scheme_and_validate_url(url.strip())
except InvalidUrl:
- LOG.warning("NEW_TAB action returned with an invalid url, skipping action", url=url)
+ LOG.warning(
+ "NEW_TAB action returned with an invalid url, skipping action",
+ **llm_response_log_metadata(url),
+ )
return NullAction(**base_action_dict)
host = urlparse(validated_url).hostname
if not host or is_blocked_host(host, resolve_dns=False):
- LOG.warning("NEW_TAB action targets a blocked host, skipping action", url=validated_url)
+ LOG.warning(
+ "NEW_TAB action targets a blocked host, skipping action",
+ **llm_response_log_metadata(validated_url),
+ )
return NullAction(**base_action_dict)
return NewTabAction(**base_action_dict, url=validated_url)
if action_type == ActionType.SWITCH_TAB:
raw_index = action.get("tab_index")
if raw_index is None:
- LOG.warning("SWITCH_TAB action returned without a tab_index, skipping action", raw_action=action)
+ LOG.warning(
+ "SWITCH_TAB action returned without a tab_index, skipping action",
+ **llm_response_log_metadata(action),
+ )
return NullAction(**base_action_dict)
try:
tab_index = int(raw_index)
except (TypeError, ValueError):
- LOG.warning("SWITCH_TAB action returned with a non-integer tab_index, skipping action", raw_action=action)
+ LOG.warning(
+ "SWITCH_TAB action returned with a non-integer tab_index, skipping action",
+ **llm_response_log_metadata(action),
+ )
return NullAction(**base_action_dict)
return SwitchTabAction(**base_action_dict, tab_index=tab_index)
@@ -392,32 +426,31 @@ def parse_actions(
LOG.warning(
"Agent decided to terminate",
task_id=task.task_id,
- llm_response=json_response,
- reasoning=action_instance.reasoning,
- actions=actions,
+ **llm_response_log_metadata(json_response),
+ action_type=type(action_instance).__name__,
)
actions.append(action_instance)
- except UnsupportedActionType:
+ except UnsupportedActionType as exc:
LOG.error(
"Unsupported action type when parsing actions",
task_id=task.task_id,
- raw_action=action,
- exc_info=True,
+ error_type=type(exc).__name__,
+ **llm_response_log_metadata(action),
)
- except (ValidationError, ValueError):
+ except (ValidationError, ValueError) as exc:
LOG.warning(
"Invalid action",
task_id=task.task_id,
- raw_action=action,
- exc_info=True,
+ error_type=type(exc).__name__,
+ **llm_response_log_metadata(action),
)
- except Exception:
+ except Exception as exc:
LOG.error(
"Failed to marshal action",
task_id=task.task_id,
- raw_action=action,
- exc_info=True,
+ error_type=type(exc).__name__,
+ **llm_response_log_metadata(action),
)
if non_object_entries:
@@ -427,7 +460,6 @@ def parse_actions(
skipped_count=len(non_object_entries),
raw_action_count=len(json_response),
parsed_action_count=len(actions),
- sample_entries=[str(entry)[:200] for entry in non_object_entries[:3]],
)
############################ This part of code might not be needed ############################
@@ -442,8 +474,8 @@ def parse_actions(
LOG.warning(
"Missing elements in scraped page",
task_id=task.task_id,
- missing_element_ids=missing_element_ids,
- all_element_ids=all_element_ids,
+ missing_element_ids_count=len(missing_element_ids),
+ all_element_ids_count=len(all_element_ids),
)
############################ This part of code might not be needed ############################
return actions
@@ -467,13 +499,15 @@ async def parse_cua_actions(
if idx < len(reasonings):
try:
reasoning = reasonings[idx].summary[0].text
- except Exception:
- LOG.exception(
+ except Exception as exc:
+ LOG.error(
"Failed to parse reasoning",
task_id=task.task_id,
step_id=step.step_id,
step_order=step.order,
action_order=idx,
+ error_type=type(exc).__name__,
+ **llm_response_log_metadata(reasonings[idx]),
)
match action_type:
@@ -547,7 +581,7 @@ async def parse_cua_actions(
step_id=step.step_id,
step_order=step.order,
action_order=idx,
- whole_path=whole_path,
+ **llm_response_log_metadata(whole_path),
)
action = WaitAction(
seconds=5,
@@ -579,13 +613,15 @@ async def parse_cua_actions(
action.step_order = step.order
action.action_order = idx
actions.append(action)
- except Exception:
- LOG.exception(
+ except Exception as exc:
+ LOG.error(
"Failed to parse action",
task_id=task.task_id,
step_id=step.step_id,
step_order=step.order,
action_order=idx,
+ error_type=type(exc).__name__,
+ **llm_response_log_metadata(cua_action),
)
break
if not actions:
@@ -595,7 +631,7 @@ async def parse_cua_actions(
step_id=step.step_id,
organization_id=task.organization_id,
workflow_run_id=task.workflow_run_id,
- response=response.model_dump(),
+ **llm_response_log_metadata(response.model_dump()),
)
reasoning = reasonings[0].summary[0].text if reasonings and reasonings[0].summary else None
assistant_message = assistant_messages[0].content[0].text if assistant_messages else None
@@ -615,13 +651,18 @@ async def parse_anthropic_actions(
) -> list[Action]:
tool_calls = [block for block in assistant_content if block["type"] == "tool_use" and block["name"] == "computer"]
reasonings = [block for block in assistant_content if block["type"] == "thinking"]
- LOG.info("Anthropic tool calls", tool_calls=tool_calls, reasonings=reasonings, assistant_content=assistant_content)
+ LOG.info(
+ "Anthropic tool calls",
+ tool_call_count=len(tool_calls),
+ reasoning_count=len(reasonings),
+ assistant_content_count=len(assistant_content),
+ )
if len(reasonings) > 1:
LOG.warning(
"Anthropic CUA: multiple reasonings in assistant content",
task_id=task.task_id,
step_id=step.step_id,
- assistant_content=assistant_content,
+ assistant_content_count=len(assistant_content),
)
reasoning = reasonings[0]["thinking"] if reasonings else None
idx = 0
@@ -640,7 +681,8 @@ async def parse_anthropic_actions(
if not coordinate:
LOG.warning(
"Anthropic CUA error: mouse move action has no coordinate",
- tool_call=tool_call,
+ tool_call_type=type(tool_call).__name__,
+ tool_call_keys_count=len(tool_call),
)
idx += 1
continue
@@ -677,7 +719,8 @@ async def parse_anthropic_actions(
if not coordinate:
LOG.warning(
"Anthropic CUA error: left click action has no coordinate and it doesn't have mouse_move before it",
- tool_call=tool_call,
+ tool_call_type=type(tool_call).__name__,
+ tool_call_keys_count=len(tool_call),
)
idx += 1
continue
@@ -719,7 +762,8 @@ async def parse_anthropic_actions(
if not text:
LOG.warning(
"Anthropic CUA error: type action has no text",
- tool_call=tool_call,
+ tool_call_type=type(tool_call).__name__,
+ tool_call_keys_count=len(tool_call),
)
idx += 1
continue
@@ -744,7 +788,8 @@ async def parse_anthropic_actions(
if not text:
LOG.warning(
"Anthropic CUA error: key action has no text",
- tool_call=tool_call,
+ tool_call_type=type(tool_call).__name__,
+ tool_call_keys_count=len(tool_call),
)
idx += 1
continue
@@ -811,7 +856,8 @@ async def parse_anthropic_actions(
else:
LOG.warning(
"Anthropic CUA error: unsupported scroll direction",
- tool_call=tool_call,
+ tool_call_type=type(tool_call).__name__,
+ tool_call_keys_count=len(tool_call),
)
idx += 1
continue
@@ -867,12 +913,19 @@ async def parse_anthropic_actions(
coordinate = tool_call_input.get("coordinate")
start_coordinate = tool_call_input.get("start_coordinate")
LOG.info(
- "Anthropic CUA left click drag action", coordinate=coordinate, start_coordinate=start_coordinate
+ "Anthropic CUA left click drag action",
+ coordinate_type=type(coordinate).__name__,
+ coordinate_length=len(coordinate) if isinstance(coordinate, (list, tuple, str)) else None,
+ start_coordinate_type=type(start_coordinate).__name__,
+ start_coordinate_length=(
+ len(start_coordinate) if isinstance(start_coordinate, (list, tuple, str)) else None
+ ),
)
if not coordinate or not start_coordinate:
LOG.warning(
"Anthropic CUA error: left click drag action has no coordinate or start coordinate",
- tool_call=tool_call,
+ tool_call_type=type(tool_call).__name__,
+ tool_call_keys_count=len(tool_call),
)
idx += 1
continue
@@ -921,15 +974,18 @@ async def parse_anthropic_actions(
else:
LOG.error(
"Anthropic CUA error: unsupported action",
- tool_call=tool_call,
+ tool_call_type=type(tool_call).__name__,
+ tool_call_keys_count=len(tool_call),
)
idx += 1
- except Exception:
- LOG.exception(
+ except Exception as exc:
+ LOG.error(
"Anthropic CUA error: failed to parse action",
task_id=task.task_id,
step_id=step.step_id,
- tool_call=tool_call,
+ tool_call_type=type(tool_call).__name__,
+ tool_call_keys_count=len(tool_call),
+ error_type=type(exc).__name__,
)
break
if not actions:
@@ -976,9 +1032,8 @@ async def generate_cua_fallback_actions(
prompt_name="cua-fallback-action",
step=step,
)
- LOG.info("Fallback action response", action_response=action_response)
+ LOG.info("Fallback action response", **llm_response_log_metadata(action_response))
skyvern_action_type = action_response.get("action")
- useful_information = action_response.get("useful_information")
# use 'other' action as fallback in the 'cua-fallback-action' prompt
# it can avoid LLM returning unreasonable actions, and fallback to use 'wait' action in agent instead
@@ -992,9 +1047,7 @@ async def generate_cua_fallback_actions(
"Updating task with useful information",
task_id=task.task_id,
organization_id=task.organization_id,
- useful_information=useful_information,
- assistant_message=assistant_message,
- reasoning=reasoning,
+ **llm_response_log_metadata(action_response),
)
await app.DATABASE.tasks.update_task(
task.task_id,
@@ -1203,12 +1256,13 @@ async def parse_ui_tars_actions(
action = _create_ui_tars_action(parsed_action, task, step, browser_window_dimension, idx)
if action:
actions.append(action)
- except Exception:
- LOG.exception(
+ except Exception as exc:
+ LOG.error(
"Failed to create UI-TARS action",
task_id=task.task_id,
step_id=step.step_id,
- parsed_action=parsed_action,
+ error_type=type(exc).__name__,
+ **llm_response_log_metadata(parsed_action),
)
continue
@@ -1217,17 +1271,18 @@ async def parse_ui_tars_actions(
"No valid actions generated from UI-TARS response",
task_id=task.task_id,
step_id=step.step_id,
- response_preview=response_content[:200],
+ **llm_response_log_metadata(response_content),
)
return actions
- except Exception:
- LOG.exception(
+ except Exception as exc:
+ LOG.error(
"Failed to parse UI-TARS actions",
task_id=task.task_id,
step_id=step.step_id,
- response_content=response_content[:200],
+ error_type=type(exc).__name__,
+ **llm_response_log_metadata(response_content),
)
return []
@@ -1307,11 +1362,11 @@ def _parse_ui_tars_response(response_content: str, browser_window_dimension: Res
parsed_action["thought"] = thought
parsed_action["browser_window_dimension"] = browser_window_dimension
parsed_actions.append(parsed_action)
- except Exception:
+ except Exception as exc:
LOG.warning(
"Failed to parse individual UI-TARS action",
- action_str=action_str,
- exc_info=True,
+ error_type=type(exc).__name__,
+ **llm_response_log_metadata(action_str),
)
continue
@@ -1351,8 +1406,12 @@ def _parse_single_action(action_str: str) -> dict[str, Any] | None:
"action_inputs": action_inputs,
}
- except Exception:
- LOG.debug(f"Failed to parse action string: {action_str}", exc_info=True)
+ except Exception as exc:
+ LOG.debug(
+ "Failed to parse action string",
+ error_type=type(exc).__name__,
+ **llm_response_log_metadata(action_str),
+ )
return None
@@ -1508,7 +1567,10 @@ def _create_ui_tars_action(
)
else:
- LOG.warning(f"Unsupported UI-TARS action type: {action_type}")
+ LOG.warning(
+ "Unsupported UI-TARS action type",
+ **llm_response_log_metadata(action_type),
+ )
return None
@@ -1581,6 +1643,10 @@ def _extract_ui_tars_coordinates(box_str: str, browser_window_dimension: Resolut
else:
return None, None
- except Exception:
- LOG.debug(f"Failed to parse UI-TARS coordinates: {box_str}", exc_info=True)
+ except Exception as exc:
+ LOG.debug(
+ "Failed to parse UI-TARS coordinates",
+ error_type=type(exc).__name__,
+ **llm_response_log_metadata(box_str),
+ )
return None, None
diff --git a/tests/unit/test_actions.py b/tests/unit/test_actions.py
index 0f0978f15..b2947f548 100644
--- a/tests/unit/test_actions.py
+++ b/tests/unit/test_actions.py
@@ -1,7 +1,7 @@
from datetime import datetime
from pathlib import Path
from typing import Any
-from unittest.mock import MagicMock
+from unittest.mock import AsyncMock, MagicMock
import pytest
from pydantic import TypeAdapter, ValidationError
@@ -47,7 +47,13 @@ from skyvern.webeye.actions.actions import (
WebAction,
)
from skyvern.webeye.actions.models import DetailedAgentStepOutput
-from skyvern.webeye.actions.parse_actions import parse_action, parse_actions
+from skyvern.webeye.actions.parse_actions import (
+ parse_action,
+ parse_actions,
+ parse_anthropic_actions,
+ parse_cua_actions,
+ parse_ui_tars_actions,
+)
def _mock_scraped_page() -> MagicMock:
@@ -57,6 +63,115 @@ def _mock_scraped_page() -> MagicMock:
return page
+async def test_parse_anthropic_actions_does_not_log_model_response_content(monkeypatch: pytest.MonkeyPatch) -> None:
+ sentinel = "private-anthropic-response-sentinel"
+ task = MagicMock(task_id="tsk_private", organization_id="org_private", workflow_run_id=None)
+ step = MagicMock(step_id="stp_private", order=0)
+ assistant_content = [
+ {"type": "thinking", "thinking": sentinel},
+ {"type": "thinking", "thinking": sentinel},
+ {
+ "type": "tool_use",
+ "name": "computer",
+ "id": "tool_private",
+ "input": {"action": "mouse_move", "private": sentinel},
+ },
+ ]
+ fallback = AsyncMock(return_value=[])
+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.generate_cua_fallback_actions", fallback)
+
+ with capture_logs() as events:
+ actions = await parse_anthropic_actions(task, step, assistant_content, None, None)
+
+ assert actions == []
+ assert sentinel not in repr(events)
+ assert fallback.await_count == 1
+
+
+def test_parse_actions_does_not_log_malformed_response_content(monkeypatch: pytest.MonkeyPatch) -> None:
+ sentinel = "private-malformed-action-sentinel"
+ context = MagicMock(totp_codes={})
+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.skyvern_context.ensure_context", lambda: context)
+ task = MagicMock(
+ task_id="tsk_private",
+ organization_id="org_private",
+ workflow_run_id=None,
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+
+ with capture_logs() as events:
+ actions = parse_actions(
+ task,
+ "stp_private",
+ 0,
+ _mock_scraped_page(),
+ [
+ {"action_type": "INPUT_TEXT", "id": "e1", "reasoning": sentinel},
+ {"action_type": "SWITCH_TAB", "tab_index": sentinel},
+ sentinel,
+ ],
+ )
+
+ assert len(actions) == 1
+ assert isinstance(actions[0], NullAction)
+ assert sentinel not in repr(events)
+ assert not any("exc_info" in event for event in events)
+
+
+async def test_parse_cua_actions_does_not_log_malformed_response_content(monkeypatch: pytest.MonkeyPatch) -> None:
+ sentinel = "private-cua-action-sentinel"
+ task = MagicMock(task_id="tsk_private", organization_id="org_private", workflow_run_id=None)
+ step = MagicMock(step_id="stp_private", order=0)
+ response = MagicMock()
+ response.output = [MagicMock(type="computer_call", action=MagicMock(type=sentinel))]
+ response.model_dump.return_value = {"output": [{"type": "computer_call", "action": {"type": sentinel}}]}
+ fallback = AsyncMock(return_value=[])
+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.generate_cua_fallback_actions", fallback)
+
+ with capture_logs() as events:
+ actions = await parse_cua_actions(task, step, response)
+
+ assert actions == []
+ assert sentinel not in repr(events)
+ assert not any("exc_info" in event for event in events)
+
+
+ ("response_content", "sentinel", "expected_event"),
+ [
+ ("Action: private_ui_tars_sentinel((", "private_ui_tars_sentinel", "Failed to parse action string"),
+ ("Action: private_ui_tars_sentinel()", "private_ui_tars_sentinel", "Unsupported UI-TARS action type"),
+ (
+ "Action: click(start_box='private-ui-tars-sentinel')",
+ "private-ui-tars-sentinel",
+ "Failed to parse UI-TARS coordinates",
+ ),
+ ],
+ ids=["malformed-action", "unsupported-action-type", "malformed-coordinates"],
+)
+async def test_parse_ui_tars_actions_does_not_log_malformed_response_content(
+ response_content: str,
+ sentinel: str,
+ expected_event: str,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ task = MagicMock(task_id="tsk_private", organization_id="org_private", workflow_run_id=None)
+ step = MagicMock(step_id="stp_private", order=0)
+ logger = MagicMock()
+ monkeypatch.setattr("skyvern.webeye.actions.parse_actions.LOG", logger)
+
+ actions = await parse_ui_tars_actions(task, step, response_content, {"width": 1280, "height": 720})
+
+ assert actions == []
+ assert expected_event in repr(logger.mock_calls)
+ assert sentinel not in repr(logger.mock_calls)
+ assert "exc_info" not in repr(logger.mock_calls)
+
+
def test_sdk_input_text_action_type_constant_matches_sdk_enum() -> None:
assert SDK_INPUT_TEXT_ACTION_TYPE == SdkActionType.AI_INPUT_TEXT.value
diff --git a/tests/unit/test_background_task_executor.py b/tests/unit/test_background_task_executor.py
index 632f30799..c1c62b978 100644
--- a/tests/unit/test_background_task_executor.py
+++ b/tests/unit/test_background_task_executor.py
@@ -2627,7 +2627,6 @@ async def test_retry_resumer_recovers_attempt_lookup_failure_on_next_sweep(
read_run.side_effect = None
dispatch.side_effect = real_entry
await executor._recover_pending_retries_once(resume_fresh=False)
- assert key in executor._scheduled_retry_resumes
await asyncio.gather(*executor._background_tasks)
execute.assert_awaited_once()
assert execute.await_args.kwargs["attempt_number"] == 2
diff --git a/tests/unit/test_error_detection_service.py b/tests/unit/test_error_detection_service.py
index a62028d53..5b38c1da1 100644
--- a/tests/unit/test_error_detection_service.py
+++ b/tests/unit/test_error_detection_service.py
@@ -8,6 +8,7 @@ from datetime import datetime
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from structlog.testing import capture_logs
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.services.error_detection_service import detect_user_defined_errors_for_task
@@ -149,6 +150,32 @@ async def test_detect_errors_no_browser_state(mock_step):
assert detected_errors[0].error_code == "payment_failed"
+async def test_context_detection_does_not_log_malformed_llm_error_payload(mock_step):
+ now = datetime.now()
+ organization = make_organization(now)
+ task = make_task(now, organization, error_code_mapping={"payment_failed": "Credit card was declined"})
+ sentinel = "private-malformed-error-sentinel"
+
+ with patch("skyvern.services.error_detection_service.app") as mock_app, capture_logs() as events:
+ mock_app.EXTRACTION_LLM_API_HANDLER = AsyncMock(
+ return_value={
+ "errors": [
+ {"error_code": sentinel, "reasoning": sentinel, "confidence_float": sentinel},
+ ]
+ }
+ )
+ detected = await detect_user_defined_errors_for_task(
+ task=task, step=mock_step, browser_state=None, failure_reason=None
+ )
+
+ assert detected == []
+ event = next(event for event in events if event["event"] == "Failed to validate user-defined error")
+ assert event["error_type"] == "ValidationError"
+ assert "exc_info" not in event
+ assert sentinel not in repr(event)
+
+
@pytest.mark.asyncio
async def test_detect_errors_no_working_page(mock_step):
"""Test that detection uses context-based method when there's no working page."""
diff --git a/tests/unit/test_file_upload_prompt.py b/tests/unit/test_file_upload_prompt.py
index f44aa33e0..8fdac307e 100644
--- a/tests/unit/test_file_upload_prompt.py
+++ b/tests/unit/test_file_upload_prompt.py
@@ -410,11 +410,13 @@ async def test_prompt_selecting_zero_files_completes_and_records_empty_output(
"workflow_run_block_id": "workflow-run-block-id",
"candidate_count": 3,
"selected_count": 0,
- "reasoning": "No candidates match.",
+ "reasoning_type": "str",
+ "reasoning_length": len("No candidates match."),
}
warning_event = zero_selection_warnings[0]
for field, value in expected_fields.items():
assert warning_event[field] == value
+ assert "No candidates match." not in repr(warning_event)
@pytest.mark.asyncio
diff --git a/tests/unit/test_handle_terminate_action.py b/tests/unit/test_handle_terminate_action.py
index b272113f5..527b32b0e 100644
--- a/tests/unit/test_handle_terminate_action.py
+++ b/tests/unit/test_handle_terminate_action.py
@@ -8,6 +8,7 @@ from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from structlog.testing import capture_logs
from skyvern.errors.errors import TimeoutGetTOTPVerificationCodeError, UserDefinedError
from skyvern.webeye.actions.actions import TerminateAction
@@ -182,6 +183,41 @@ async def test_terminate_preserves_errors_when_extract_fails() -> None:
assert action.errors[0].error_code == "OTP_TIMEOUT"
+async def test_terminate_does_not_log_malformed_llm_error_payload(monkeypatch: pytest.MonkeyPatch) -> None:
+ sentinel = "private-malformed-terminate-error-sentinel"
+ task = _make_task(error_code_mapping={"LOGIN_FAILED": "the login did not complete"})
+ task.navigation_goal = "Log in"
+ task.navigation_payload = {}
+ action = TerminateAction(reasoning="Page validation failed", errors=[])
+
+ with (
+ _surface_errors_llm_returns(
+ monkeypatch,
+ [{"error_code": sentinel, "reasoning": sentinel, "confidence_float": sentinel}],
+ ),
+ capture_logs() as events,
+ ):
+ results = await handle_terminate_action(
+ action=action,
+ page=MagicMock(),
+ scraped_page=_make_scraped_page(),
+ task=task,
+ step=_make_step(),
+ )
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionSuccess)
+ event = next(
+ event
+ for event in events
+ if event["event"] == "extract_user_defined_errors failed, using errors from action reasoning"
+ )
+ assert event["error_type"] == "ValidationError"
+ assert "exc_info" not in event
+ assert sentinel not in repr(event)
+
+
@pytest.mark.asyncio
async def test_extract_user_defined_errors_falls_back_to_reasoning_match(monkeypatch: pytest.MonkeyPatch) -> None:
task = _make_task(error_code_mapping={"portal_inaccessible": "portal is inaccessible"})
diff --git a/tests/unit/test_llm_response_parsing.py b/tests/unit/test_llm_response_parsing.py
index 389f5467b..6000d853b 100644
--- a/tests/unit/test_llm_response_parsing.py
+++ b/tests/unit/test_llm_response_parsing.py
@@ -42,6 +42,22 @@ def test_coerce_response_to_dict_variants(response, expected):
assert expected[1]
+ ("response", "expected"),
+ [
+ ({"llm_response": "private-sentinel"}, {"response_type": "dict", "response_keys_count": 1}),
+ (["private-sentinel"], {"response_type": "list", "response_length": 1}),
+ ("private-sentinel", {"response_type": "str", "response_length": 16}),
+ (None, {"response_type": "NoneType"}),
+ ],
+)
+def test_llm_response_log_metadata_contains_only_shape(response, expected):
+ metadata = llm_utils.llm_response_log_metadata(response)
+
+ assert metadata == expected
+ assert "private-sentinel" not in repr(metadata)
+
+
def test_coerce_response_warning_keeps_metadata_without_response_content():
sentinel = "private-response-sentinel"
response = [{"reasoning": sentinel}]
@@ -53,6 +69,11 @@ def test_coerce_response_warning_keeps_metadata_without_response_content():
assert event["response_type"] == "list"
assert event["response_length"] == 1
assert event["prompt_name"] == "extract-actions"
+ first_item_event = next(
+ event for event in events if event["event"] == "Parsed LLM response is a list; using first dict element"
+ )
+ assert first_item_event["first_item_keys_count"] == 1
+ assert "first_item_keys" not in first_item_event
assert all(sentinel not in repr(event) for event in events)
diff --git a/tests/unit/test_task_v2_summary_envelope.py b/tests/unit/test_task_v2_summary_envelope.py
index 52c962575..fdfb9aac4 100644
--- a/tests/unit/test_task_v2_summary_envelope.py
+++ b/tests/unit/test_task_v2_summary_envelope.py
@@ -13,6 +13,7 @@ from datetime import timezone
from types import SimpleNamespace
import pytest
+from structlog.testing import capture_logs
from skyvern.forge.prompts import prompt_engine
from skyvern.services import task_v2_service
@@ -67,6 +68,20 @@ def test_parser_recovers_top_level_payload() -> None:
assert output == _NON_ENVELOPE_RESPONSE
+def test_parser_recovery_logs_only_key_count() -> None:
+ response = {"private-model-key": "private-model-value"}
+
+ with capture_logs() as events:
+ assert _parse_task_v2_summary_response(response) == (None, response)
+
+ event = next(
+ event for event in events if event["event"] == "Recovered task_v2 summary output from non-envelope response"
+ )
+ assert event["recovered_keys_count"] == 1
+ assert "private-model-key" not in repr(event)
+ assert "private-model-value" not in repr(event)
+
+
def test_parser_recovery_preserves_user_schema_description_key() -> None:
# A goal-defined contract may itself use "description"; recovery must not strip it.
resp = {"description": "Widget", "price": 10}
diff --git a/tests/unit/test_text_prompt_block.py b/tests/unit/test_text_prompt_block.py
index 9bbc32fdd..2926d21a0 100644
--- a/tests/unit/test_text_prompt_block.py
+++ b/tests/unit/test_text_prompt_block.py
@@ -864,6 +864,26 @@ def _patch_send_prompt_deps(monkeypatch, *, handler) -> None:
monkeypatch.setattr(block_module.asyncio, "sleep", AsyncMock())
+async def test_text_prompt_response_log_omits_llm_output(monkeypatch):
+ from structlog.testing import capture_logs
+
+ sentinel = "private-llm-response-sentinel"
+
+ async def fake_llm(**kwargs):
+ return {"llm_response": sentinel}
+
+ _patch_send_prompt_deps(monkeypatch, handler=fake_llm)
+ block = _make_text_prompt_block(prompt="Return a synthetic response.")
+
+ with capture_logs() as events:
+ result = await block.send_prompt(block.prompt, workflow_run_id="wr_qa", organization_id="o_qa")
+
+ assert result == {"llm_response": sentinel}
+ response_log = next(event for event in events if event["event"] == "TextPromptBlock Received response from LLM")
+ assert sentinel not in repr(response_log)
+
+
@pytest.mark.asyncio
async def test_text_prompt_block_retries_transient_llm_provider_error(monkeypatch):
"""A transient provider error retries with backoff and recovers instead of failing."""