Skip to content

Automate browser based workflows with AI

fix(SKY-8065): improve magic link tab close behavior with prompt fix and code fallback (#5715)

LawyZhengb46f9bdde4bd
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 3d3602b1c..62d6f07e5 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -439,6 +439,19 @@ class AgentFunction:
Cloud overrides this to True and provides the Temporal-backed implementations below.
"""
+ # Phrases that indicate a magic-link confirmation page meant to be closed.
+ # Keep lowercase; matching is case-insensitive.
+ MAGIC_LINK_CLOSE_SIGNALS: tuple[str, ...] = (
+ "close this page",
+ "close this tab",
+ "close this window",
+ "you can close",
+ "you may now close",
+ "safe to close",
+ "return to the original page",
+ "return to the original tab",
+ )
+
def get_mcp_oauth_issuer_url(self) -> str | None:
"""Return the cloud OAuth issuer URL when the build provides one.
@@ -537,7 +550,74 @@ class AgentFunction:
return None
async def post_step_execution(self, task: Task, step: Step) -> None:
- return
+ if step.status == StepStatus.completed:
+ await self._maybe_close_magic_link_page(task)
+
+ async def _maybe_close_magic_link_page(self, task: Task) -> None:
+ """Close a magic-link confirmation page if it shows close/return signals.
+
+ Some magic-link flows open a new tab for verification.
+ After the user clicks "Allow", the page says "close this page and return
+ to the original page". The LLM may miss the CLOSE_PAGE action, leaving
+ the tab open. This fallback detects such confirmation pages and closes
+ them so subsequent workflow blocks see the original page.
+ """
+ context = skyvern_context.current()
+ if not context:
+ return
+
+ if not context.has_magic_link_page(task.task_id):
+ return
+
+ page = context.magic_link_pages[task.task_id]
+
+ try:
+ visible_text = (await page.inner_text("body", timeout=5000)).lower()
+ except Exception:
+ LOG.warning(
+ "Failed to read magic link page content, skipping auto-close",
+ task_id=task.task_id,
+ exc_info=True,
+ )
+ return
+
+ matched_signal = next(
+ (signal for signal in self.MAGIC_LINK_CLOSE_SIGNALS if signal in visible_text),
+ None,
+ )
+ if matched_signal is None:
+ LOG.debug(
+ "Magic link page does not contain close signals, keeping open",
+ task_id=task.task_id,
+ page_url=page.url,
+ )
+ return
+
+ LOG.info(
+ "Magic link confirmation page detected, auto-closing",
+ task_id=task.task_id,
+ page_url=page.url,
+ matched_signal=matched_signal,
+ )
+ try:
+ async with asyncio.timeout(5):
+ await page.close()
+ except Exception:
+ # Intentionally keep the stale reference so the next completed step
+ # retries the close. Eventually page.is_closed() will return True
+ # and the entry will be cleaned up at the top of this method.
+ LOG.warning(
+ "Failed to close magic link page, will retry on next step",
+ task_id=task.task_id,
+ exc_info=True,
+ )
+ return
+
+ context.magic_link_pages.pop(task.task_id, None)
+ LOG.info(
+ "Magic link page closed successfully",
+ task_id=task.task_id,
+ )
async def post_cache_step_execution(self, task: Task, step: Step) -> None:
return
diff --git a/skyvern/forge/prompts/skyvern/extract-action-static.j2 b/skyvern/forge/prompts/skyvern/extract-action-static.j2
index 7ed2cbbf6..bd40d018e 100644
--- a/skyvern/forge/prompts/skyvern/extract-action-static.j2
+++ b/skyvern/forge/prompts/skyvern/extract-action-static.j2
@@ -18,7 +18,7 @@ Reply in JSON format with the following keys:
"user_detail_query": str, // Think of this value as a Jeopardy question and the intention behind the action. Ask the user for the details you need for executing this action. IMPORTANT: This question must be user information agnostic and must NOT contain specific user data (like names, IDs, specific values, etc.). Ask what information is needed generically. Ask the question even if the details are disclosed in user goal or user details. If it's a text field, ask for the text. If it's a file upload, ask for the file. If it's a dropdown, ask for the relevant information. If you are clicking on something specific, ask about what the intention is behind the click and what to click on. If you're downloading a file and you have multiple options, ask the user which one to download. Examples are: "What product ID should I input into the search bar?", "What file should I upload?", "What is the previous insurance provider of the user?", "Which invoice should I download?", "Does the user have any pets?", "Which member should I take action on?". NEVER include specific user data in this question - keep it generic so it can be answered with different user contexts.
"user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details. This answer CAN contain specific user data.
"confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
- "action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL"{{', "CLOSE_PAGE"' if has_magic_link_page else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.{{' "CLOSE_PAGE" is used to close the current page when it is impossible to achieve the user goal on the current page.' if has_magic_link_page else ''}}
+ "action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL"{{', "CLOSE_PAGE"' if has_magic_link_page else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.{{' "CLOSE_PAGE" is used to close the current page when the page instructs you to close it or return to the original page, OR when it is impossible to achieve the user goal on the current page.' if has_magic_link_page else ''}}
"id": str, // The id of the element to take action on. The id has to be one from the elements list. For target-less actions (SOLVE_CAPTCHA, WAIT, COMPLETE, TERMINATE, KEYPRESS), set this to null.
"captcha_type": str, // The type of captcha for SOLVE_CAPTCHA action only. null if not SOLVE_CAPTCHA action. It's a string enum: "TEXT_CAPTCHA", "RECAPTCHA", "HCAPTCHA", "MTCAPTCHA", "FUNCAPTCHA", "CLOUDFLARE", "OTHER".
"text": str, // Text for INPUT_TEXT action only
diff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2
index 1f84c75ee..0f6ca6a7e 100644
--- a/skyvern/forge/prompts/skyvern/extract-action.j2
+++ b/skyvern/forge/prompts/skyvern/extract-action.j2
@@ -18,7 +18,7 @@ Reply in JSON format with the following keys:
"user_detail_query": str, // Think of this value as a Jeopardy question and the intention behind the action. Ask the user for the details you need for executing this action. IMPORTANT: This question must be user information agnostic and must NOT contain specific user data (like names, IDs, specific values, etc.). Ask what information is needed generically. Ask the question even if the details are disclosed in user goal or user details. If it's a text field, ask for the text. If it's a file upload, ask for the file. If it's a dropdown, ask for the relevant information. If you are clicking on something specific, ask about what the intention is behind the click and what to click on. If you're downloading a file and you have multiple options, ask the user which one to download. Examples are: "What product ID should I input into the search bar?", "What file should I upload?", "What is the previous insurance provider of the user?", "Which invoice should I download?", "Does the user have any pets?", "Which member should I take action on?". NEVER include specific user data in this question - keep it generic so it can be answered with different user contexts.
"user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details. This answer CAN contain specific user data.
"confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
- "action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL"{{', "CLOSE_PAGE"' if has_magic_link_page else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.{{' "CLOSE_PAGE" is used to close the current page when it is impossible to achieve the user goal on the current page.' if has_magic_link_page else ''}}
+ "action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL"{{', "CLOSE_PAGE"' if has_magic_link_page else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.{{' "CLOSE_PAGE" is used to close the current page when the page instructs you to close it or return to the original page, OR when it is impossible to achieve the user goal on the current page.' if has_magic_link_page else ''}}
"id": str, // The id of the element to take action on. The id has to be one from the elements list. For target-less actions (SOLVE_CAPTCHA, WAIT, COMPLETE, TERMINATE, KEYPRESS), set this to null.
"captcha_type": str, // The type of captcha for SOLVE_CAPTCHA action only. null if not SOLVE_CAPTCHA action. It's a string enum: "TEXT_CAPTCHA", "RECAPTCHA", "HCAPTCHA", "MTCAPTCHA", "FUNCAPTCHA", "CLOUDFLARE", "OTHER".
"text": str, // Text for INPUT_TEXT action only
diff --git a/tests/unit/test_magic_link_auto_close.py b/tests/unit/test_magic_link_auto_close.py
new file mode 100644
index 000000000..b52bcf36f
--- /dev/null
+++ b/tests/unit/test_magic_link_auto_close.py
@@ -0,0 +1,157 @@
+"""Tests for AgentFunction._maybe_close_magic_link_page fallback."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.forge.agent_functions import AgentFunction
+from skyvern.forge.sdk.models import StepStatus
+
+
+def _make_step(*, status: StepStatus = StepStatus.completed) -> MagicMock:
+ step = MagicMock()
+ step.status = status
+ return step
+
+
+def _make_task(task_id: str = "tsk_test") -> MagicMock:
+ task = MagicMock()
+ task.task_id = task_id
+ return task
+
+
+def _make_page(*, closed: bool = False, inner_text: str = "") -> MagicMock:
+ page = MagicMock()
+ page.is_closed.return_value = closed
+ page.url = "https://example.com/confirmation"
+ page.inner_text = AsyncMock(return_value=inner_text)
+ page.close = AsyncMock()
+ return page
+
+
+def _make_context(task_id: str, page: MagicMock | None, *, has_page: bool | None = None) -> MagicMock:
+ ctx = MagicMock()
+ ctx.magic_link_pages = {task_id: page} if page else {}
+ # Default: has_page = True if page is provided and not closed
+ if has_page is None:
+ has_page = page is not None and not page.is_closed()
+ ctx.has_magic_link_page.return_value = has_page
+ return ctx
+
+
+async def test_no_context() -> None:
+ """Should return silently when no SkyvernContext exists."""
+ agent_fn = AgentFunction()
+ with patch("skyvern.forge.agent_functions.skyvern_context") as mock_ctx:
+ mock_ctx.current.return_value = None
+ await agent_fn._maybe_close_magic_link_page(_make_task())
+ # No exception raised
+
+
+async def test_no_magic_link_page() -> None:
+ """Should return silently when no magic link page is tracked for this task."""
+ agent_fn = AgentFunction()
+ ctx = _make_context("tsk_test", None)
+ with patch("skyvern.forge.agent_functions.skyvern_context") as mock_ctx:
+ mock_ctx.current.return_value = ctx
+ await agent_fn._maybe_close_magic_link_page(_make_task())
+
+
+async def test_page_already_closed() -> None:
+ """Should return early when has_magic_link_page returns False (page already closed)."""
+ agent_fn = AgentFunction()
+ page = _make_page(closed=True)
+ ctx = _make_context("tsk_test", page) # has_page defaults to False for closed pages
+ with patch("skyvern.forge.agent_functions.skyvern_context") as mock_ctx:
+ mock_ctx.current.return_value = ctx
+ await agent_fn._maybe_close_magic_link_page(_make_task())
+ page.close.assert_not_called()
+
+
+async def test_no_close_signal() -> None:
+ """Should keep the page open when no close signal is found."""
+ agent_fn = AgentFunction()
+ page = _make_page(inner_text="Welcome to the dashboard. Here are your settings.")
+ ctx = _make_context("tsk_test", page)
+ with patch("skyvern.forge.agent_functions.skyvern_context") as mock_ctx:
+ mock_ctx.current.return_value = ctx
+ await agent_fn._maybe_close_magic_link_page(_make_task())
+ assert "tsk_test" in ctx.magic_link_pages
+ page.close.assert_not_called()
+
+
+async def test_close_signal_matched() -> None:
+ """Should close the page and clean up context when a signal matches."""
+ agent_fn = AgentFunction()
+ page = _make_page(
+ inner_text="Account verification successful. You can close this page and return to the original page."
+ )
+ ctx = _make_context("tsk_test", page)
+ with patch("skyvern.forge.agent_functions.skyvern_context") as mock_ctx:
+ mock_ctx.current.return_value = ctx
+ await agent_fn._maybe_close_magic_link_page(_make_task())
+ assert "tsk_test" not in ctx.magic_link_pages
+ page.close.assert_called_once()
+
+
+async def test_close_signal_case_insensitive() -> None:
+ """Signal matching should be case-insensitive."""
+ agent_fn = AgentFunction()
+ page = _make_page(inner_text="You May Now Close This Tab")
+ ctx = _make_context("tsk_test", page)
+ with patch("skyvern.forge.agent_functions.skyvern_context") as mock_ctx:
+ mock_ctx.current.return_value = ctx
+ await agent_fn._maybe_close_magic_link_page(_make_task())
+ assert "tsk_test" not in ctx.magic_link_pages
+ page.close.assert_called_once()
+
+
+async def test_inner_text_failure() -> None:
+ """Should skip auto-close when reading page text fails."""
+ agent_fn = AgentFunction()
+ page = _make_page()
+ page.inner_text = AsyncMock(side_effect=Exception("page crashed"))
+ ctx = _make_context("tsk_test", page)
+ with patch("skyvern.forge.agent_functions.skyvern_context") as mock_ctx:
+ mock_ctx.current.return_value = ctx
+ await agent_fn._maybe_close_magic_link_page(_make_task())
+ assert "tsk_test" in ctx.magic_link_pages
+ page.close.assert_not_called()
+
+
+async def test_page_close_failure_keeps_reference() -> None:
+ """Should keep the stale reference for retry when page.close() fails."""
+ agent_fn = AgentFunction()
+ page = _make_page(inner_text="You can close this page now.")
+ page.close = AsyncMock(side_effect=Exception("close failed"))
+ ctx = _make_context("tsk_test", page)
+ with patch("skyvern.forge.agent_functions.skyvern_context") as mock_ctx:
+ mock_ctx.current.return_value = ctx
+ await agent_fn._maybe_close_magic_link_page(_make_task())
+ # Reference kept for retry on next step
+ assert "tsk_test" in ctx.magic_link_pages
+
+
[email protected]("status", [StepStatus.failed, StepStatus.created])
+async def test_skipped_on_non_completed_step(status: StepStatus) -> None:
+ """Should skip magic link check when step is not completed."""
+ agent_fn = AgentFunction()
+ page = _make_page(inner_text="You can close this page now.")
+ ctx = _make_context("tsk_test", page)
+ with patch("skyvern.forge.agent_functions.skyvern_context") as mock_ctx:
+ mock_ctx.current.return_value = ctx
+ await agent_fn.post_step_execution(_make_task(), _make_step(status=status))
+ # Page should NOT be closed — step didn't complete
+ assert "tsk_test" in ctx.magic_link_pages
+ page.close.assert_not_called()