Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: persist real LLM cost to DB + block-level attribution (public API unchanged — placeholder preserved) (#5656)
pedrohsdb3fce82760fd1
diff --git a/alembic/versions/2026_04_24_2353-c19d7d385560_add_llm_cost_to_workflow_run_blocks.py b/alembic/versions/2026_04_24_2353-c19d7d385560_add_llm_cost_to_workflow_run_blocks.pynew file mode 100644index 000000000..11e48eebf--- /dev/null+++ b/alembic/versions/2026_04_24_2353-c19d7d385560_add_llm_cost_to_workflow_run_blocks.py@@ -0,0 +1,30 @@+"""add llm_cost to workflow_run_blocks++Revision ID: c19d7d385560+Revises: c9005bafa5ec+Create Date: 2026-04-24T23:53:46.912017+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "c19d7d385560"+down_revision: Union[str, None] = "c9005bafa5ec"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ op.add_column(+ "workflow_run_blocks",+ sa.Column("llm_cost", sa.Numeric(), nullable=False, server_default="0"),+ )+++def downgrade() -> None:+ op.drop_column("workflow_run_blocks", "llm_cost")diff --git a/skyvern/forge/sdk/api/llm/api_handler.py b/skyvern/forge/sdk/api/llm/api_handler.pyindex 1d0305389..48fa152be 100644--- a/skyvern/forge/sdk/api/llm/api_handler.py+++ b/skyvern/forge/sdk/api/llm/api_handler.py@@ -15,6 +15,7 @@ class LLMAPIHandler(Protocol):task_v2: TaskV2 | None = None,thought: Thought | None = None,ai_suggestion: AISuggestion | None = None,+ workflow_run_block_id: str | None = None,screenshots: list[bytes] | None = None,parameters: dict[str, Any] | None = None,organization_id: str | None = None,@@ -34,6 +35,7 @@ async def dummy_llm_api_handler(task_v2: TaskV2 | None = None,thought: Thought | None = None,ai_suggestion: AISuggestion | None = None,+ workflow_run_block_id: str | None = None,screenshots: list[bytes] | None = None,parameters: dict[str, Any] | None = None,organization_id: str | None = None,diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex 004a6c1b8..e8f3445ff 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -222,13 +222,45 @@ def _normalize_llm_model(model: str | None) -> str | None:return model.split("/")[-1]-def _assert_step_thought_exclusive(step: Step | None, thought: Thought | None) -> None:- # step and thought write the same llm_cost to different tables- # (steps.step_cost vs observer_thoughts.thought_cost). int_org_llm_costs- # UNION ALLs them, so setting both would double-count cost in- # fct_org_margin.llm_cost.- if step is not None and thought is not None:- raise ValueError("LLM API handler invoked with both step and thought set — these are mutually exclusive")+def _assert_step_thought_block_exclusive(+ step: Step | None,+ thought: Thought | None,+ workflow_run_block_id: str | None,+) -> None:+ # Each LLM call writes cost to exactly one of: steps.step_cost,+ # observer_thoughts.thought_cost, workflow_run_blocks.llm_cost.+ # Both the run-level SUM and the int_org_llm_costs dbt model rely on+ # this exclusivity to avoid double-counting.+ set_count = sum(1 for x in (step, thought, workflow_run_block_id) if x is not None)+ if set_count > 1:+ raise ValueError(+ "LLM API handler invoked with more than one of step / thought / workflow_run_block_id set — "+ "these are mutually exclusive"+ )+++async def _persist_block_llm_cost(+ workflow_run_block_id: str,+ organization_id: str | None,+ context: skyvern_context.SkyvernContext | None,+ llm_cost: float,+ prompt_name: str | None,+) -> None:+ """Increment workflow_run_blocks.llm_cost or warn if no org_id resolves."""+ block_org_id = organization_id or (context.organization_id if context else None)+ if block_org_id:+ await app.DATABASE.observer.increment_workflow_run_block_llm_cost(+ workflow_run_block_id=workflow_run_block_id,+ organization_id=block_org_id,+ amount=llm_cost,+ )+ else:+ LOG.warning(+ "Block LLM cost dropped: workflow_run_block_id set but no organization_id resolved",+ workflow_run_block_id=workflow_run_block_id,+ llm_cost=llm_cost,+ prompt_name=prompt_name,+ )def _convert_allowed_fails_policy(policy: LLMAllowedFailsPolicy | None) -> AllowedFailsPolicy | None:@@ -562,6 +594,7 @@ class LLMAPIHandlerFactory:task_v2: TaskV2 | None = None,thought: Thought | None = None,ai_suggestion: AISuggestion | None = None,+ workflow_run_block_id: str | None = None,screenshots: list[bytes] | None = None,parameters: dict[str, Any] | None = None,organization_id: str | None = None,@@ -584,7 +617,7 @@ class LLMAPIHandlerFactory:Returns:The response from the LLM router."""- _assert_step_thought_exclusive(step, thought)+ _assert_step_thought_block_exclusive(step, thought, workflow_run_block_id)start_time = time.perf_counter()_llm_span = otel_trace.get_current_span()_llm_span.set_attribute("llm_key", llm_key)@@ -1000,6 +1033,12 @@ class LLMAPIHandlerFactory:cached_token_count=cached_tokens if cached_tokens > 0 else None,last_llm_model=actual_model,)+ if workflow_run_block_id:+ # Atomic UPDATE: description gen (asyncio.create_task in+ # execute_safe) races with the block's own execute() calls.+ await _persist_block_llm_cost(+ workflow_run_block_id, organization_id, context, llm_cost, prompt_name+ )parsed_response = parse_api_response(response, llm_config.add_assistant_prefix, force_dict)parsed_response_json = json.dumps(parsed_response, indent=2)if should_persist_llm_artifacts:@@ -1154,6 +1193,7 @@ class LLMAPIHandlerFactory:task_v2: TaskV2 | None = None,thought: Thought | None = None,ai_suggestion: AISuggestion | None = None,+ workflow_run_block_id: str | None = None,screenshots: list[bytes] | None = None,parameters: dict[str, Any] | None = None,organization_id: str | None = None,@@ -1164,7 +1204,7 @@ class LLMAPIHandlerFactory:force_dict: bool = True,system_prompt: str | None = None,) -> dict[str, Any] | Any:- _assert_step_thought_exclusive(step, thought)+ _assert_step_thought_block_exclusive(step, thought, workflow_run_block_id)start_time = time.perf_counter()_llm_span = otel_trace.get_current_span()# handler_type distinguishes the three LLM entry points that share@@ -1527,6 +1567,10 @@ class LLMAPIHandlerFactory:thought_cost=llm_cost,last_llm_model=actual_model,)+ if workflow_run_block_id:+ await _persist_block_llm_cost(+ workflow_run_block_id, organization_id, context, llm_cost, prompt_name+ )parsed_response = parse_api_response(response, llm_config.add_assistant_prefix, force_dict)parsed_response_json = json.dumps(parsed_response, indent=2)if should_persist_llm_artifacts:@@ -1749,6 +1793,7 @@ class LLMCaller:task_v2: TaskV2 | None = None,thought: Thought | None = None,ai_suggestion: AISuggestion | None = None,+ workflow_run_block_id: str | None = None,screenshots: list[bytes] | None = None,parameters: dict[str, Any] | None = None,organization_id: str | None = None,@@ -1760,7 +1805,7 @@ class LLMCaller:system_prompt: str | None = None,**extra_parameters: Any,) -> dict[str, Any] | Any:- _assert_step_thought_exclusive(step, thought)+ _assert_step_thought_block_exclusive(step, thought, workflow_run_block_id)start_time = time.perf_counter()_llm_span = otel_trace.get_current_span()_llm_span.set_attribute("llm_key", self.llm_key)@@ -1996,6 +2041,12 @@ class LLMCaller:thought_cost=call_stats.llm_cost,last_llm_model=actual_model,)+ if workflow_run_block_id and call_stats and call_stats.llm_cost is not None:+ # call_stats.llm_cost is None when litellm can't compute cost+ # (volcengine, some OPENAI_COMPATIBLE targets).+ await _persist_block_llm_cost(+ workflow_run_block_id, organization_id, context, call_stats.llm_cost, prompt_name+ )organization_id = organization_id or (step.organization_id if step else (thought.organization_id if thought else None)diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex de2ea1091..707f5a8e4 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -824,6 +824,9 @@ class WorkflowRunBlockModel(Base):executed_branch_result = Column(Boolean, nullable=True)executed_branch_next_block = Column(String, nullable=True)+ # Accumulates LLM cost for block-scoped calls (no step/thought attribution).+ llm_cost = Column(Numeric, default=0, nullable=False)+created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False)diff --git a/skyvern/forge/sdk/db/repositories/observer.py b/skyvern/forge/sdk/db/repositories/observer.pyindex da44a54c1..fdb1a3ca9 100644--- a/skyvern/forge/sdk/db/repositories/observer.py+++ b/skyvern/forge/sdk/db/repositories/observer.py@@ -3,7 +3,8 @@ from __future__ import annotationsfrom datetime import datetime, timezonefrom typing import TYPE_CHECKING, Any, Callable-from sqlalchemy import and_, delete, select+import structlog+from sqlalchemy import and_, delete, func, select, updatefrom sqlalchemy.exc import SQLAlchemyErrorfrom skyvern.forge.sdk.db._error_handling import db_operation@@ -30,6 +31,8 @@ from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlockfrom skyvern.schemas.runs import ProxyLocationInput, RunEnginefrom skyvern.schemas.workflows import BlockStatus, BlockType+LOG = structlog.get_logger()+class ObserverRepository(BaseRepository):"""Database operations for observer tasks (TaskV2), thoughts, and workflow run blocks."""@@ -120,6 +123,64 @@ class ObserverRepository(BaseRepository):thoughts = (await session.scalars(query)).all()return [Thought.model_validate(thought) for thought in thoughts]+ @db_operation("get_thought_cost_sum_by_workflow_run_id")+ async def get_thought_cost_sum_by_workflow_run_id(self, workflow_run_id: str, organization_id: str) -> float:+ """Sum `thought_cost` across all thoughts for the given workflow_run_id.++ Returns 0.0 for runs without task_v2 planning.+ """+ async with self.Session() as session:+ query = (+ select(func.coalesce(func.sum(ThoughtModel.thought_cost), 0))+ .where(ThoughtModel.workflow_run_id == workflow_run_id)+ .where(ThoughtModel.organization_id == organization_id)+ )+ total = (await session.execute(query)).scalar_one()+ return float(total)++ @db_operation("get_block_llm_cost_sum_by_workflow_run_id")+ async def get_block_llm_cost_sum_by_workflow_run_id(self, workflow_run_id: str, organization_id: str) -> float:+ """Sum `llm_cost` across all workflow_run_blocks for this workflow_run_id."""+ async with self.Session() as session:+ query = (+ select(func.coalesce(func.sum(WorkflowRunBlockModel.llm_cost), 0))+ .where(WorkflowRunBlockModel.workflow_run_id == workflow_run_id)+ .where(WorkflowRunBlockModel.organization_id == organization_id)+ )+ total = (await session.execute(query)).scalar_one()+ return float(total)++ @db_operation("increment_workflow_run_block_llm_cost")+ async def increment_workflow_run_block_llm_cost(+ self,+ workflow_run_block_id: str,+ organization_id: str,+ amount: float,+ ) -> None:+ """Atomically add `amount` to `workflow_run_blocks.llm_cost`.++ Single SQL UPDATE so concurrent writers don't lose increments.+ No-op for non-positive `amount`.+ """+ if amount <= 0:+ return+ async with self.Session() as session:+ stmt = (+ update(WorkflowRunBlockModel)+ .where(WorkflowRunBlockModel.workflow_run_block_id == workflow_run_block_id)+ .where(WorkflowRunBlockModel.organization_id == organization_id)+ .values(llm_cost=WorkflowRunBlockModel.llm_cost + amount)+ )+ result = await session.execute(stmt)+ await session.commit()+ if result.rowcount == 0:+ LOG.warning(+ "Block LLM cost increment matched zero rows — cost dropped",+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ amount=amount,+ )+@db_operation("create_task_v2")async def create_task_v2(self,diff --git a/skyvern/forge/sdk/db/repositories/tasks.py b/skyvern/forge/sdk/db/repositories/tasks.pyindex 4504f872d..eac12d56e 100644--- a/skyvern/forge/sdk/db/repositories/tasks.py+++ b/skyvern/forge/sdk/db/repositories/tasks.py@@ -229,6 +229,23 @@ class TasksRepository(BaseRepository):row = (await session.execute(query)).one()return row.total, row.completed+ @db_operation("get_step_cost_sum_by_task_ids")+ async def get_step_cost_sum_by_task_ids(self, task_ids: list[str], organization_id: str) -> float:+ """Sum `step_cost` across all steps belonging to the given task_ids.++ Returns 0.0 for empty task_ids. Includes failed steps.+ """+ if not task_ids:+ return 0.0+ async with self.Session() as session:+ query = (+ select(func.coalesce(func.sum(StepModel.step_cost), 0))+ .where(StepModel.task_id.in_(task_ids))+ .where(StepModel.organization_id == organization_id)+ )+ total = (await session.execute(query)).scalar_one()+ return float(total)+@db_operation("get_workflow_run_progress_timestamps")async def get_workflow_run_progress_timestamps(self,diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 6419304dc..9a1eb9282 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -568,7 +568,10 @@ class Block(BaseModel, abc.ABC):block=block_data,)json_response = await app.SECONDARY_LLM_API_HANDLER(- prompt=description_generation_prompt, prompt_name="generate-workflow-run-block-description"+ prompt=description_generation_prompt,+ prompt_name="generate-workflow-run-block-description",+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,)description = json_response.get("summary")LOG.info(@@ -2833,7 +2836,12 @@ class TextPromptBlock(Block):prompt=prompt,llm_key=self.llm_key,)- response = await llm_api_handler(prompt=prompt, prompt_name="text-prompt")+ response = await llm_api_handler(+ prompt=prompt,+ prompt_name="text-prompt",+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )if workflow_run_block:artifacts_to_persist.append((ArtifactType.LLM_RESPONSE, json.dumps(response).encode("utf-8")))@@ -3980,7 +3988,12 @@ class FileParserBlock(Block):file_url=self.file_url, file_type=self.file_type, error=f"Failed to parse Excel file: {str(e)}")- async def _parse_pdf_file(self, file_path: str) -> str:+ async def _parse_pdf_file(+ self,+ file_path: str,+ workflow_run_block_id: str | None = None,+ organization_id: str | None = None,+ ) -> str:"""Parse PDF file and return extracted text.Uses the shared PDF parsing utility that tries pypdf first,@@ -4016,6 +4029,8 @@ class FileParserBlock(Block):prompt_name="extract-text-from-image",screenshots=page_images,force_dict=True,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,)return llm_response.get("extracted_text", "")except Exception:@@ -4025,7 +4040,12 @@ class FileParserBlock(Block):)raise- async def _parse_image_file(self, file_path: str) -> str:+ async def _parse_image_file(+ self,+ file_path: str,+ workflow_run_block_id: str | None = None,+ organization_id: str | None = None,+ ) -> str:"""Parse image file using vision LLM for OCR."""try:with open(file_path, "rb") as f:@@ -4040,6 +4060,8 @@ class FileParserBlock(Block):prompt_name="extract-text-from-image",screenshots=[image_bytes],force_dict=True,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,)return llm_response.get("extracted_text", "")except Exception:@@ -4117,7 +4139,11 @@ class FileParserBlock(Block):)async def _extract_with_ai(- self, content: str | list[dict[str, Any]], workflow_run_context: WorkflowRunContext+ self,+ content: str | list[dict[str, Any]],+ workflow_run_context: WorkflowRunContext,+ workflow_run_block_id: str | None = None,+ organization_id: str | None = None,) -> dict[str, Any]:"""Extract structured data using AI based on json_schema."""# Use local variable to avoid mutating the instance@@ -4146,7 +4172,11 @@ class FileParserBlock(Block):llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(llm_key, default=app.LLM_API_HANDLER)llm_response = await llm_api_handler(- prompt=llm_prompt, prompt_name="extract-information-from-file-text", force_dict=False+ prompt=llm_prompt,+ prompt_name="extract-information-from-file-text",+ force_dict=False,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,)return llm_response@@ -4223,9 +4253,17 @@ class FileParserBlock(Block):elif self.file_type == FileType.EXCEL:parsed_data = await self._parse_excel_file(file_path)elif self.file_type == FileType.PDF:- parsed_data = await self._parse_pdf_file(file_path)+ parsed_data = await self._parse_pdf_file(+ file_path,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )elif self.file_type == FileType.IMAGE:- parsed_data = await self._parse_image_file(file_path)+ parsed_data = await self._parse_image_file(+ file_path,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )elif self.file_type == FileType.DOCX:parsed_data = await self._parse_docx_file(file_path)else:@@ -4250,7 +4288,12 @@ class FileParserBlock(Block):if self.json_schema:try:- ai_extracted_data = await self._extract_with_ai(parsed_data, workflow_run_context)+ ai_extracted_data = await self._extract_with_ai(+ parsed_data,+ workflow_run_context,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )final_data = ai_extracted_dataexcept Exception as e:return await self.build_block_result(@@ -4370,7 +4413,11 @@ class PDFParserBlock(Block):"extract-information-from-file-text", extracted_text_content=extracted_text, json_schema=self.json_schema)llm_response = await app.LLM_API_HANDLER(- prompt=llm_prompt, prompt_name="extract-information-from-file-text", force_dict=False+ prompt=llm_prompt,+ prompt_name="extract-information-from-file-text",+ force_dict=False,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,)# Record the parsed dataawait self.record_output_parameter_value(workflow_run_context, workflow_run_id, llm_response)diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex ab65c744a..8ef33ad71 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -4428,6 +4428,43 @@ class WorkflowService:urls = await app.ARTIFACT_MANAGER.get_share_links_with_bundle_support(artifacts)return [u for u in urls if u is not None]+ async def get_workflow_run_llm_cost_sum(+ self,+ workflow_run_id: str,+ organization_id: str,+ ) -> float:+ """Sum per-LLM-call cost across step_cost, thought_cost, and+ workflow_run_blocks.llm_cost for this workflow_run.++ `organization_id` is required: passing None makes repo filters+ evaluate as `IS NULL` and silently return 0.0.+ """+ if not organization_id:+ raise ValueError(+ "get_workflow_run_llm_cost_sum requires organization_id; "+ "passing None would compile to IS NULL and silently return 0.0"+ )+ # thought + block sums are independent of the task list; run them in+ # parallel with the task fetch + step sum (which depends on task ids).+ thought_task = app.DATABASE.observer.get_thought_cost_sum_by_workflow_run_id(+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ )+ block_task = app.DATABASE.observer.get_block_llm_cost_sum_by_workflow_run_id(+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ )+ workflow_run_tasks = await app.DATABASE.tasks.get_tasks_by_workflow_run_id(workflow_run_id=workflow_run_id)+ step_cost_sum, thought_cost_sum, block_llm_cost_sum = await asyncio.gather(+ app.DATABASE.tasks.get_step_cost_sum_by_task_ids(+ task_ids=[task.task_id for task in workflow_run_tasks],+ organization_id=organization_id,+ ),+ thought_task,+ block_task,+ )+ return step_cost_sum + thought_cost_sum + block_llm_cost_sum+async def build_workflow_run_status_response_by_workflow_id(self,workflow_run_id: str,@@ -4600,7 +4637,7 @@ class WorkflowService:text_prompt_blocks = [block for block in workflow_run_blocks if block.block_type == BlockType.TEXT_PROMPT]- # TODO: This is a temporary cost calculation. We need to implement a more accurate cost calculation.+ # This is a temporary cost calculation.total_cost = 0.05 * (completed_step_count + len(text_prompt_blocks))return WorkflowRunResponseBase(workflow_id=workflow.workflow_permanent_id,diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.pyindex 8c61ed58b..abe633403 100644--- a/tests/unit/test_api_handler_factory.py+++ b/tests/unit/test_api_handler_factory.py@@ -236,15 +236,21 @@ def test_normalize_llm_model_strips_provider_prefix() -> None:assert api_handler_factory._normalize_llm_model(None) is None-def test_assert_step_thought_exclusive_rejects_both_set() -> None:+def test_assert_step_thought_block_exclusive_rejects_both_set() -> None:with pytest.raises(ValueError, match="mutually exclusive"):- api_handler_factory._assert_step_thought_exclusive(MagicMock(), MagicMock())+ api_handler_factory._assert_step_thought_block_exclusive(MagicMock(), MagicMock(), None)-def test_assert_step_thought_exclusive_allows_single_or_neither() -> None:- api_handler_factory._assert_step_thought_exclusive(None, None)- api_handler_factory._assert_step_thought_exclusive(MagicMock(), None)- api_handler_factory._assert_step_thought_exclusive(None, MagicMock())+def test_assert_step_thought_block_exclusive_rejects_step_and_block() -> None:+ with pytest.raises(ValueError, match="mutually exclusive"):+ api_handler_factory._assert_step_thought_block_exclusive(MagicMock(), None, "wfb_123")+++def test_assert_step_thought_block_exclusive_allows_single_or_neither() -> None:+ api_handler_factory._assert_step_thought_block_exclusive(None, None, None)+ api_handler_factory._assert_step_thought_block_exclusive(MagicMock(), None, None)+ api_handler_factory._assert_step_thought_block_exclusive(None, MagicMock(), None)+ api_handler_factory._assert_step_thought_block_exclusive(None, None, "wfb_123")@pytest.mark.asynciodiff --git a/tests/unit/test_block_llm_cost_kwargs_wiring.py b/tests/unit/test_block_llm_cost_kwargs_wiring.pynew file mode 100644index 000000000..e406edefe--- /dev/null+++ b/tests/unit/test_block_llm_cost_kwargs_wiring.py@@ -0,0 +1,101 @@+"""AST guard: every awaited LLM handler call inside the six+block-scoped methods below must pass both `workflow_run_block_id=`+and `organization_id=`. Catches call-site wiring regressions.+"""++from __future__ import annotations++import ast+import pathlib++BLOCK_PY = (+ pathlib.Path(__file__).resolve().parents[2] / "skyvern" / "forge" / "sdk" / "workflow" / "models" / "block.py"+)++# Methods that make block-scoped LLM calls. Each must pass+# workflow_run_block_id= and organization_id= to any LLM handler call+# in its body so the handler can associate the call with the block.+BLOCK_ATTRIBUTED_METHODS = {+ "_generate_workflow_run_block_description", # BaseBlock description gen+ "send_prompt", # TextPromptBlock+ "_parse_pdf_file", # FileParserBlock PDF vision+ "_parse_image_file", # FileParserBlock image OCR+ "_extract_with_ai", # FileParserBlock schema extract+ "execute", # PDFParserBlock (deprecated) execute+}++# Names that indicate an LLM handler call (both module-level handlers and+# locally-bound `llm_api_handler` from LLMAPIHandlerFactory.get_override_llm_api_handler).+LLM_HANDLER_NAME_FRAGMENTS = ("LLM_API_HANDLER", "llm_api_handler")+++def _is_llm_handler_call(call: ast.Call) -> bool:+ """True if the call looks like `await (whatever).LLM_API_HANDLER(...)` or+ `await llm_api_handler(...)`."""+ func = call.func+ if isinstance(func, ast.Attribute):+ return any(frag in func.attr for frag in LLM_HANDLER_NAME_FRAGMENTS)+ if isinstance(func, ast.Name):+ return any(frag in func.id for frag in LLM_HANDLER_NAME_FRAGMENTS)+ return False+++def _find_llm_calls_in_method(method: ast.AsyncFunctionDef) -> list[ast.Call]:+ """Collect LLM handler INVOCATIONS (awaited calls) only — not factory+ lookups like `LLMAPIHandlerFactory.get_override_llm_api_handler(...)`+ which happen to contain `llm_api_handler` in the name but aren't the+ thing that makes an LLM request."""+ calls: list[ast.Call] = []+ for node in ast.walk(method):+ if not isinstance(node, ast.Await):+ continue+ inner = node.value+ if isinstance(inner, ast.Call) and _is_llm_handler_call(inner):+ calls.append(inner)+ return calls+++def _kwarg_names(call: ast.Call) -> set[str]:+ return {kw.arg for kw in call.keywords if kw.arg is not None}+++def test_every_block_scoped_llm_call_passes_both_cost_attribution_kwargs() -> None:+ tree = ast.parse(BLOCK_PY.read_text())++ offenders: list[str] = []+ methods_with_calls: set[str] = set()++ for node in ast.walk(tree):+ if not isinstance(node, ast.AsyncFunctionDef):+ continue+ if node.name not in BLOCK_ATTRIBUTED_METHODS:+ continue++ for call in _find_llm_calls_in_method(node):+ methods_with_calls.add(node.name)+ kwargs = _kwarg_names(call)+ missing = {"workflow_run_block_id", "organization_id"} - kwargs+ if missing:+ offenders.append(+ f"{node.name} @ line {call.lineno}: missing {sorted(missing)} "+ f"on LLM handler call (kwargs present: {sorted(kwargs)})"+ )++ # Sanity: every method in BLOCK_ATTRIBUTED_METHODS must have at least one+ # awaited LLM call. If our name-fragment matcher misses (e.g. a handler+ # rename), the kwargs check would silently pass with zero calls inspected.+ assert methods_with_calls == BLOCK_ATTRIBUTED_METHODS, (+ f"Expected every method in BLOCK_ATTRIBUTED_METHODS to contain at least one "+ f"awaited LLM handler call. Methods with no calls found: "+ f"{sorted(BLOCK_ATTRIBUTED_METHODS - methods_with_calls)}. "+ f"Either the matcher needs a new name fragment, or the method no longer "+ f"makes a block-scoped LLM call (remove from BLOCK_ATTRIBUTED_METHODS)."+ )++ assert not offenders, (+ "Block-scoped LLM calls are missing cost-attribution kwargs:\n "+ + "\n ".join(offenders)+ + "\n\nEvery LLM handler call inside these methods must pass both "+ "`workflow_run_block_id=` and `organization_id=` to correctly attribute "+ "cost to `workflow_run_blocks.llm_cost`."+ )