Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: extend feature flag resolution logging to standalone tasks (#5541)
Shuchang Zheng9650df43f901
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex 52feb3e9f..feb6bad37 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -37,7 +37,7 @@ class SkyvernContext:max_screenshot_scrolls: int | None = Nonebrowser_container_ip: str | None = Nonebrowser_container_task_arn: str | None = None- workflow_feature_flags_entries: dict[str, bool | str | None] = field(default_factory=dict)+ feature_flag_entries: dict[str, bool | str | None] = field(default_factory=dict)# feature flagsenable_page_ready_wait: bool = False@@ -123,35 +123,45 @@ class SkyvernContext:return Falsereturn True- def flush_workflow_feature_flags(self) -> None:- if not self.workflow_feature_flags_entries:+ def flush_feature_flags(self) -> None:+ if not self.feature_flag_entries:return- if not self.workflow_run_id:++ has_workflow = bool(self.workflow_run_id)+ has_task = bool(self.task_id or self.task_v2_id or self.run_id)++ if not (has_workflow or has_task):LOG.debug(- "Discarding workflow_feature_flags entries for non-workflow context",- count=len(self.workflow_feature_flags_entries),+ "Discarding feature flag entries for non-run context",+ count=len(self.feature_flag_entries),)- self.workflow_feature_flags_entries.clear()+ self.feature_flag_entries.clear()return- feature_resolutions = dict(sorted(self.workflow_feature_flags_entries.items()))+ feature_resolutions = dict(sorted(self.feature_flag_entries.items()))log_fields: dict[str, Any] = {"organization_id": str(self.organization_id or ""),- "workflow_run_id": str(self.workflow_run_id),- "workflow_permanent_id": str(self.workflow_permanent_id or ""),"feature_resolutions": feature_resolutions,"service_name": settings.OTEL_SERVICE_NAME,}+ if self.workflow_run_id:+ log_fields["workflow_run_id"] = str(self.workflow_run_id)+ if self.workflow_permanent_id:+ log_fields["workflow_permanent_id"] = str(self.workflow_permanent_id)if self.task_id:log_fields["task_id"] = str(self.task_id)if self.task_v2_id:log_fields["task_v2_id"] = str(self.task_v2_id)+ if self.run_id:+ log_fields["run_id"] = str(self.run_id)if self.browser_session_id:log_fields["browser_session_id"] = str(self.browser_session_id)if self.request_id:log_fields["request_id"] = str(self.request_id)- LOG.info("workflow_feature_flags", **log_fields)- self.workflow_feature_flags_entries.clear()++ event_name = "workflow_feature_flags" if has_workflow else "task_feature_flags"+ LOG.info(event_name, **log_fields)+ self.feature_flag_entries.clear()_context: ContextVar[SkyvernContext | None] = ContextVar(@@ -209,13 +219,13 @@ def replace(context: SkyvernContext) -> None:Returns:None"""- _flush_workflow_feature_flags_if_needed(current())+ _flush_feature_flags_if_needed(current())_context.set(context)-def _flush_workflow_feature_flags_if_needed(context: SkyvernContext | None) -> None:- if context is not None and context.workflow_feature_flags_entries:- context.flush_workflow_feature_flags()+def _flush_feature_flags_if_needed(context: SkyvernContext | None) -> None:+ if context is not None and context.feature_flag_entries:+ context.flush_feature_flags()def _restore(token: Token[SkyvernContext | None]) -> None:@@ -228,7 +238,7 @@ def _restore(token: Token[SkyvernContext | None]) -> None:Returns:None"""- _flush_workflow_feature_flags_if_needed(current())+ _flush_feature_flags_if_needed(current())_context.reset(token)@@ -257,5 +267,5 @@ def reset() -> None:Returns:None"""- _flush_workflow_feature_flags_if_needed(current())+ _flush_feature_flags_if_needed(current())_context.set(None)diff --git a/skyvern/forge/sdk/experimentation/providers.py b/skyvern/forge/sdk/experimentation/providers.pyindex cefaa6dc5..a5e49fcb6 100644--- a/skyvern/forge/sdk/experimentation/providers.py+++ b/skyvern/forge/sdk/experimentation/providers.py@@ -37,11 +37,11 @@ def _serialize_feature_resolution_value(resolution_kind: ResolutionKind, resolvereturn str(resolved_value)-def _should_emit_workflow_feature_flags(context: skyvern_context.SkyvernContext) -> bool:- return bool(context.workflow_run_id)+def _should_record_feature_flags(context: skyvern_context.SkyvernContext) -> bool:+ return bool(context.workflow_run_id or context.task_id or context.task_v2_id or context.run_id)-def _record_workflow_feature_flags(+def _record_feature_flag_entry(context: skyvern_context.SkyvernContext | None,*,feature_name: str,@@ -49,10 +49,10 @@ def _record_workflow_feature_flags() -> None:if context is None:return- if not _should_emit_workflow_feature_flags(context):+ if not _should_record_feature_flags(context):return- context.workflow_feature_flags_entries[feature_name] = resolved_value+ context.feature_flag_entries[feature_name] = resolved_valuedef _build_feature_flag_resolution_log_fields(@@ -87,10 +87,10 @@ def _build_feature_flag_resolution_log_fields(return log_fields-def flush_workflow_feature_flags(context: skyvern_context.SkyvernContext | None = None) -> None:+def flush_feature_flags(context: skyvern_context.SkyvernContext | None = None) -> None:resolved_context = context or skyvern_context.current()if resolved_context is not None:- resolved_context.flush_workflow_feature_flags()+ resolved_context.flush_feature_flags()def record_feature_flag_resolution(@@ -110,7 +110,7 @@ def record_feature_flag_resolution(resolved_value=resolved_value,),)- _record_workflow_feature_flags(+ _record_feature_flag_entry(context,feature_name=feature_name,resolved_value=serialized_resolved_value,diff --git a/tests/unit/test_feature_flag_resolution_logging.py b/tests/unit/test_feature_flag_resolution_logging.pyindex a2a717978..cacac2cde 100644--- a/tests/unit/test_feature_flag_resolution_logging.py+++ b/tests/unit/test_feature_flag_resolution_logging.py@@ -460,7 +460,7 @@ def test_request_only_context_does_not_emit_workflow_feature_flags(assert [event for _, event, _ in logger.records if event == "workflow_feature_flags"] == []-def test_task_only_context_does_not_emit_workflow_feature_flags(+def test_task_only_context_emits_task_feature_flags(monkeypatch: pytest.MonkeyPatch,) -> None:logger = CaptureLogger()@@ -478,4 +478,89 @@ def test_task_only_context_does_not_emit_workflow_feature_flags(debug_fields = [fields for _, event, fields in logger.records if event == "feature_flag_resolution"][0]assert debug_fields["task_id"] == "tsk_only"assert debug_fields["organization_id"] == "org_123"+assert [event for _, event, _ in logger.records if event == "workflow_feature_flags"] == []+ task_summaries = [fields for _, event, fields in logger.records if event == "task_feature_flags"]+ assert len(task_summaries) == 1+ summary = task_summaries[0]+ assert summary["task_id"] == "tsk_only"+ assert summary["organization_id"] == "org_123"+ assert summary["feature_resolutions"] == {"TEST_FLAG": False}+ assert "workflow_run_id" not in summary+ assert "workflow_permanent_id" not in summary+++def test_task_v2_only_context_emits_task_feature_flags(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ logger = CaptureLogger()+ _set_logger(monkeypatch, logger)+ skyvern_context.set(SkyvernContext(task_v2_id="tv2_only", organization_id="org_123"))++ providers_module.record_feature_flag_resolution(+ feature_name="TEST_FLAG",+ resolution_kind="value",+ resolved_value="variant-a",+ )++ skyvern_context.reset()++ task_summaries = [fields for _, event, fields in logger.records if event == "task_feature_flags"]+ assert len(task_summaries) == 1+ summary = task_summaries[0]+ assert summary["task_v2_id"] == "tv2_only"+ assert summary["feature_resolutions"] == {"TEST_FLAG": "variant-a"}+ assert "workflow_run_id" not in summary+ assert [event for _, event, _ in logger.records if event == "workflow_feature_flags"] == []+++def test_workflow_with_task_emits_only_workflow_feature_flags(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ logger = CaptureLogger()+ _set_logger(monkeypatch, logger)+ skyvern_context.set(+ SkyvernContext(+ organization_id="org_123",+ workflow_run_id="wr_123",+ workflow_permanent_id="wfp_123",+ task_id="tsk_123",+ )+ )++ providers_module.record_feature_flag_resolution(+ feature_name="TEST_FLAG",+ resolution_kind="enabled",+ resolved_value=True,+ )++ skyvern_context.reset()++ workflow_summaries = [fields for _, event, fields in logger.records if event == "workflow_feature_flags"]+ task_summaries = [fields for _, event, fields in logger.records if event == "task_feature_flags"]+ assert len(workflow_summaries) == 1+ assert len(task_summaries) == 0+ assert workflow_summaries[0]["workflow_run_id"] == "wr_123"+ assert workflow_summaries[0]["task_id"] == "tsk_123"+ assert workflow_summaries[0]["feature_resolutions"] == {"TEST_FLAG": True}+++def test_run_id_only_context_emits_task_feature_flags(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ logger = CaptureLogger()+ _set_logger(monkeypatch, logger)+ skyvern_context.set(SkyvernContext(run_id="run_only", organization_id="org_123"))++ providers_module.record_feature_flag_resolution(+ feature_name="TEST_FLAG",+ resolution_kind="enabled",+ resolved_value=True,+ )++ skyvern_context.reset()++ task_summaries = [fields for _, event, fields in logger.records if event == "task_feature_flags"]+ assert len(task_summaries) == 1+ assert task_summaries[0]["run_id"] == "run_only"+ assert task_summaries[0]["feature_resolutions"] == {"TEST_FLAG": True}diff --git a/tests/unit/test_taskv2_block_context_preservation.py b/tests/unit/test_taskv2_block_context_preservation.pyindex 0c34a20a9..8e71cfa4b 100644--- a/tests/unit/test_taskv2_block_context_preservation.py+++ b/tests/unit/test_taskv2_block_context_preservation.py@@ -108,6 +108,7 @@ def test_scoped_child_context_restores_parent_and_flushes_child_summary(monkeypa"workflow_run_id": "wr_child","workflow_permanent_id": "wfp_child","task_v2_id": "tsk_v2_child",+ "run_id": "wr_parent","feature_resolutions": {"TEST_FLAG": True},"service_name": settings.OTEL_SERVICE_NAME,},@@ -163,6 +164,7 @@ def test_scoped_child_summary_does_not_fragment_parent_summary(monkeypatch: pyte"workflow_run_id": "wr_child","workflow_permanent_id": "wfp_child","task_v2_id": "tsk_v2_child",+ "run_id": "wr_parent","feature_resolutions": {"CHILD_FLAG": False},"service_name": settings.OTEL_SERVICE_NAME,},@@ -174,6 +176,7 @@ def test_scoped_child_summary_does_not_fragment_parent_summary(monkeypatch: pyte"organization_id": "org_parent","workflow_run_id": "wr_parent","workflow_permanent_id": "wfp_parent",+ "run_id": "wr_parent","feature_resolutions": {"PARENT_AFTER": False,"PARENT_BEFORE": True,@@ -217,6 +220,7 @@ def test_replace_flushes_existing_context_before_overwrite(monkeypatch: pytest.M"organization_id": "org_original","workflow_run_id": "wr_original","workflow_permanent_id": "wfp_original",+ "run_id": "wr_original","feature_resolutions": {"ORIGINAL_FLAG": "variant-a"},"service_name": settings.OTEL_SERVICE_NAME,},