Skip to content

Automate browser based workflows with AI

feat: extend feature flag resolution logging to standalone tasks (#5541)

Shuchang Zheng9650df43f901
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index 52feb3e9f..feb6bad37 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -37,7 +37,7 @@ class SkyvernContext:
max_screenshot_scrolls: int | None = None
browser_container_ip: str | None = None
browser_container_task_arn: str | None = None
- workflow_feature_flags_entries: dict[str, bool | str | None] = field(default_factory=dict)
+ feature_flag_entries: dict[str, bool | str | None] = field(default_factory=dict)
# feature flags
enable_page_ready_wait: bool = False
@@ -123,35 +123,45 @@ class SkyvernContext:
return False
return True
- def flush_workflow_feature_flags(self) -> None:
- if not self.workflow_feature_flags_entries:
+ def flush_feature_flags(self) -> None:
+ if not self.feature_flag_entries:
return
- if not self.workflow_run_id:
+
+ has_workflow = bool(self.workflow_run_id)
+ has_task = bool(self.task_id or self.task_v2_id or self.run_id)
+
+ if not (has_workflow or has_task):
LOG.debug(
- "Discarding workflow_feature_flags entries for non-workflow context",
- count=len(self.workflow_feature_flags_entries),
+ "Discarding feature flag entries for non-run context",
+ count=len(self.feature_flag_entries),
)
- self.workflow_feature_flags_entries.clear()
+ self.feature_flag_entries.clear()
return
- feature_resolutions = dict(sorted(self.workflow_feature_flags_entries.items()))
+ feature_resolutions = dict(sorted(self.feature_flag_entries.items()))
log_fields: dict[str, Any] = {
"organization_id": str(self.organization_id or ""),
- "workflow_run_id": str(self.workflow_run_id),
- "workflow_permanent_id": str(self.workflow_permanent_id or ""),
"feature_resolutions": feature_resolutions,
"service_name": settings.OTEL_SERVICE_NAME,
}
+ if self.workflow_run_id:
+ log_fields["workflow_run_id"] = str(self.workflow_run_id)
+ if self.workflow_permanent_id:
+ log_fields["workflow_permanent_id"] = str(self.workflow_permanent_id)
if self.task_id:
log_fields["task_id"] = str(self.task_id)
if self.task_v2_id:
log_fields["task_v2_id"] = str(self.task_v2_id)
+ if self.run_id:
+ log_fields["run_id"] = str(self.run_id)
if self.browser_session_id:
log_fields["browser_session_id"] = str(self.browser_session_id)
if self.request_id:
log_fields["request_id"] = str(self.request_id)
- LOG.info("workflow_feature_flags", **log_fields)
- self.workflow_feature_flags_entries.clear()
+
+ event_name = "workflow_feature_flags" if has_workflow else "task_feature_flags"
+ LOG.info(event_name, **log_fields)
+ self.feature_flag_entries.clear()
_context: ContextVar[SkyvernContext | None] = ContextVar(
@@ -209,13 +219,13 @@ def replace(context: SkyvernContext) -> None:
Returns:
None
"""
- _flush_workflow_feature_flags_if_needed(current())
+ _flush_feature_flags_if_needed(current())
_context.set(context)
-def _flush_workflow_feature_flags_if_needed(context: SkyvernContext | None) -> None:
- if context is not None and context.workflow_feature_flags_entries:
- context.flush_workflow_feature_flags()
+def _flush_feature_flags_if_needed(context: SkyvernContext | None) -> None:
+ if context is not None and context.feature_flag_entries:
+ context.flush_feature_flags()
def _restore(token: Token[SkyvernContext | None]) -> None:
@@ -228,7 +238,7 @@ def _restore(token: Token[SkyvernContext | None]) -> None:
Returns:
None
"""
- _flush_workflow_feature_flags_if_needed(current())
+ _flush_feature_flags_if_needed(current())
_context.reset(token)
@@ -257,5 +267,5 @@ def reset() -> None:
Returns:
None
"""
- _flush_workflow_feature_flags_if_needed(current())
+ _flush_feature_flags_if_needed(current())
_context.set(None)
diff --git a/skyvern/forge/sdk/experimentation/providers.py b/skyvern/forge/sdk/experimentation/providers.py
index cefaa6dc5..a5e49fcb6 100644
--- a/skyvern/forge/sdk/experimentation/providers.py
+++ b/skyvern/forge/sdk/experimentation/providers.py
@@ -37,11 +37,11 @@ def _serialize_feature_resolution_value(resolution_kind: ResolutionKind, resolve
return str(resolved_value)
-def _should_emit_workflow_feature_flags(context: skyvern_context.SkyvernContext) -> bool:
- return bool(context.workflow_run_id)
+def _should_record_feature_flags(context: skyvern_context.SkyvernContext) -> bool:
+ return bool(context.workflow_run_id or context.task_id or context.task_v2_id or context.run_id)
-def _record_workflow_feature_flags(
+def _record_feature_flag_entry(
context: skyvern_context.SkyvernContext | None,
*,
feature_name: str,
@@ -49,10 +49,10 @@ def _record_workflow_feature_flags(
) -> None:
if context is None:
return
- if not _should_emit_workflow_feature_flags(context):
+ if not _should_record_feature_flags(context):
return
- context.workflow_feature_flags_entries[feature_name] = resolved_value
+ context.feature_flag_entries[feature_name] = resolved_value
def _build_feature_flag_resolution_log_fields(
@@ -87,10 +87,10 @@ def _build_feature_flag_resolution_log_fields(
return log_fields
-def flush_workflow_feature_flags(context: skyvern_context.SkyvernContext | None = None) -> None:
+def flush_feature_flags(context: skyvern_context.SkyvernContext | None = None) -> None:
resolved_context = context or skyvern_context.current()
if resolved_context is not None:
- resolved_context.flush_workflow_feature_flags()
+ resolved_context.flush_feature_flags()
def record_feature_flag_resolution(
@@ -110,7 +110,7 @@ def record_feature_flag_resolution(
resolved_value=resolved_value,
),
)
- _record_workflow_feature_flags(
+ _record_feature_flag_entry(
context,
feature_name=feature_name,
resolved_value=serialized_resolved_value,
diff --git a/tests/unit/test_feature_flag_resolution_logging.py b/tests/unit/test_feature_flag_resolution_logging.py
index a2a717978..cacac2cde 100644
--- a/tests/unit/test_feature_flag_resolution_logging.py
+++ b/tests/unit/test_feature_flag_resolution_logging.py
@@ -460,7 +460,7 @@ def test_request_only_context_does_not_emit_workflow_feature_flags(
assert [event for _, event, _ in logger.records if event == "workflow_feature_flags"] == []
-def test_task_only_context_does_not_emit_workflow_feature_flags(
+def test_task_only_context_emits_task_feature_flags(
monkeypatch: pytest.MonkeyPatch,
) -> None:
logger = CaptureLogger()
@@ -478,4 +478,89 @@ def test_task_only_context_does_not_emit_workflow_feature_flags(
debug_fields = [fields for _, event, fields in logger.records if event == "feature_flag_resolution"][0]
assert debug_fields["task_id"] == "tsk_only"
assert debug_fields["organization_id"] == "org_123"
+
assert [event for _, event, _ in logger.records if event == "workflow_feature_flags"] == []
+ task_summaries = [fields for _, event, fields in logger.records if event == "task_feature_flags"]
+ assert len(task_summaries) == 1
+ summary = task_summaries[0]
+ assert summary["task_id"] == "tsk_only"
+ assert summary["organization_id"] == "org_123"
+ assert summary["feature_resolutions"] == {"TEST_FLAG": False}
+ assert "workflow_run_id" not in summary
+ assert "workflow_permanent_id" not in summary
+
+
+def test_task_v2_only_context_emits_task_feature_flags(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ logger = CaptureLogger()
+ _set_logger(monkeypatch, logger)
+ skyvern_context.set(SkyvernContext(task_v2_id="tv2_only", organization_id="org_123"))
+
+ providers_module.record_feature_flag_resolution(
+ feature_name="TEST_FLAG",
+ resolution_kind="value",
+ resolved_value="variant-a",
+ )
+
+ skyvern_context.reset()
+
+ task_summaries = [fields for _, event, fields in logger.records if event == "task_feature_flags"]
+ assert len(task_summaries) == 1
+ summary = task_summaries[0]
+ assert summary["task_v2_id"] == "tv2_only"
+ assert summary["feature_resolutions"] == {"TEST_FLAG": "variant-a"}
+ assert "workflow_run_id" not in summary
+ assert [event for _, event, _ in logger.records if event == "workflow_feature_flags"] == []
+
+
+def test_workflow_with_task_emits_only_workflow_feature_flags(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ logger = CaptureLogger()
+ _set_logger(monkeypatch, logger)
+ skyvern_context.set(
+ SkyvernContext(
+ organization_id="org_123",
+ workflow_run_id="wr_123",
+ workflow_permanent_id="wfp_123",
+ task_id="tsk_123",
+ )
+ )
+
+ providers_module.record_feature_flag_resolution(
+ feature_name="TEST_FLAG",
+ resolution_kind="enabled",
+ resolved_value=True,
+ )
+
+ skyvern_context.reset()
+
+ workflow_summaries = [fields for _, event, fields in logger.records if event == "workflow_feature_flags"]
+ task_summaries = [fields for _, event, fields in logger.records if event == "task_feature_flags"]
+ assert len(workflow_summaries) == 1
+ assert len(task_summaries) == 0
+ assert workflow_summaries[0]["workflow_run_id"] == "wr_123"
+ assert workflow_summaries[0]["task_id"] == "tsk_123"
+ assert workflow_summaries[0]["feature_resolutions"] == {"TEST_FLAG": True}
+
+
+def test_run_id_only_context_emits_task_feature_flags(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ logger = CaptureLogger()
+ _set_logger(monkeypatch, logger)
+ skyvern_context.set(SkyvernContext(run_id="run_only", organization_id="org_123"))
+
+ providers_module.record_feature_flag_resolution(
+ feature_name="TEST_FLAG",
+ resolution_kind="enabled",
+ resolved_value=True,
+ )
+
+ skyvern_context.reset()
+
+ task_summaries = [fields for _, event, fields in logger.records if event == "task_feature_flags"]
+ assert len(task_summaries) == 1
+ assert task_summaries[0]["run_id"] == "run_only"
+ assert task_summaries[0]["feature_resolutions"] == {"TEST_FLAG": True}
diff --git a/tests/unit/test_taskv2_block_context_preservation.py b/tests/unit/test_taskv2_block_context_preservation.py
index 0c34a20a9..8e71cfa4b 100644
--- a/tests/unit/test_taskv2_block_context_preservation.py
+++ b/tests/unit/test_taskv2_block_context_preservation.py
@@ -108,6 +108,7 @@ def test_scoped_child_context_restores_parent_and_flushes_child_summary(monkeypa
"workflow_run_id": "wr_child",
"workflow_permanent_id": "wfp_child",
"task_v2_id": "tsk_v2_child",
+ "run_id": "wr_parent",
"feature_resolutions": {"TEST_FLAG": True},
"service_name": settings.OTEL_SERVICE_NAME,
},
@@ -163,6 +164,7 @@ def test_scoped_child_summary_does_not_fragment_parent_summary(monkeypatch: pyte
"workflow_run_id": "wr_child",
"workflow_permanent_id": "wfp_child",
"task_v2_id": "tsk_v2_child",
+ "run_id": "wr_parent",
"feature_resolutions": {"CHILD_FLAG": False},
"service_name": settings.OTEL_SERVICE_NAME,
},
@@ -174,6 +176,7 @@ def test_scoped_child_summary_does_not_fragment_parent_summary(monkeypatch: pyte
"organization_id": "org_parent",
"workflow_run_id": "wr_parent",
"workflow_permanent_id": "wfp_parent",
+ "run_id": "wr_parent",
"feature_resolutions": {
"PARENT_AFTER": False,
"PARENT_BEFORE": True,
@@ -217,6 +220,7 @@ def test_replace_flushes_existing_context_before_overwrite(monkeypatch: pytest.M
"organization_id": "org_original",
"workflow_run_id": "wr_original",
"workflow_permanent_id": "wfp_original",
+ "run_id": "wr_original",
"feature_resolutions": {"ORIGINAL_FLAG": "variant-a"},
"service_name": settings.OTEL_SERVICE_NAME,
},