Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Admit a blog route removal only behind a reviewed 301 redirect (SKY-15847) (#8496)
Shuchang Zheng46fc8f0d817f
diff --git a/skyvern/forge/sdk/api/files.py b/skyvern/forge/sdk/api/files.pyindex aa37a0a29..34b55335e 100644--- a/skyvern/forge/sdk/api/files.py+++ b/skyvern/forge/sdk/api/files.py@@ -248,6 +248,22 @@ class GuardedFileResponse:GuardedFileFetchHopResult: TypeAlias = GuardedFileRedirect | GuardedFileResponse+def _origin_authorizable_url(url: str) -> str:+ """Return *url* with query/fragment backslashes percent-encoded so the origin guard does not+ refuse a benign Windows-style path carried there. Scheme, authority, and path keep their+ backslashes so this stays fail-closed on an authority-parse divergence; at the download seam that+ is defense-in-depth, because validate_and_pin (AnyHttpUrl/WHATWG) already normalizes authority and+ path backslashes before the origin check. Only for the origin check — never the fetched URL."""+ cut = len(url)+ for separator in ("?", "#"):+ index = url.find(separator)+ if index != -1 and index < cut:+ cut = index+ if cut == len(url):+ return url+ return url[:cut] + url[cut:].replace("\\", "%5C")++async def fetch_file_bytes(url: str,*,@@ -258,6 +274,7 @@ async def fetch_file_bytes(authorize_request_hop: RedirectHopAuthorizer[GuardedFileFetchHopResult],download_scope: str | None = None,approved_initial_url: str | None = None,+ normalize_query_backslashes: bool = False,) -> GuardedFileResponse:"""Fetch a bounded HTTP file through the validated, pinned, per-hop authorization seam.@@ -272,17 +289,28 @@ async def fetch_file_bytes(resolver = SSRFGuardedResolver()current_url = await validate_and_pin_fetch_url(url, resolver)- if canonicalize_origin(current_url) is None:+ origin_source = _origin_authorizable_url(current_url) if normalize_query_backslashes else current_url+ if canonicalize_origin(origin_source) is None:raise HttpException(400, "[redacted]", "URL has no browser-canonicalizable HTTP origin")if allowed_redirect_origin is not None and _url_origin(current_url) != _url_origin(allowed_redirect_origin):raise HttpException(400, "[redacted]", "Cross-origin redirect blocked by policy")+ # The run-scoped authorizer canonicalizes target_url/initial_url via canonicalize_effect_target,+ # which refuses a backslash exactly like the precheck above. Present it the same query-normalized+ # form; current_url below stays raw so the encoded wire request and SSRF pinning are unchanged.+ authz_initial_url = (+ _origin_authorizable_url(approved_initial_url)+ if normalize_query_backslashes and approved_initial_url is not None+ else approved_initial_url+ )+request_headers = dict(headers or {})source_url: str | None = Nonemax_size_bytes = max_size_mb * 1024 * 1024async with aiohttp.ClientSession(connector=ssrf_guarded_tcp_connector(resolver)) as session:for _ in range(MAX_SAFE_REDIRECTS + 1):encoded_url = encode_url(current_url)+ authz_target_url = _origin_authorizable_url(current_url) if normalize_query_backslashes else current_urlasync def dispatch(_resolved_values: tuple[str, ...]) -> GuardedFileFetchHopResult:async with session.get(@@ -312,10 +340,10 @@ async def fetch_file_bytes(authorize_request_hop,RedirectHopAuthorization(source_url=source_url,- target_url=current_url,+ target_url=authz_target_url,method="GET",download_scope=download_scope,- initial_url=approved_initial_url,+ initial_url=authz_initial_url,),dispatch,)@@ -323,7 +351,8 @@ async def fetch_file_bytes(return resultnext_url = await validate_and_pin_redirect_url(current_url, result.location, resolver)- if canonicalize_origin(next_url) is None:+ next_origin_source = _origin_authorizable_url(next_url) if normalize_query_backslashes else next_url+ if canonicalize_origin(next_origin_source) is None:raise HttpException(400, "[redacted]", "Redirect has no browser-canonicalizable HTTP origin")if allowed_redirect_origin is not None and _url_origin(next_url) != _url_origin(allowed_redirect_origin):raise HttpException(400, "[redacted]", "Cross-origin redirect blocked by policy")diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 99d70afd7..aea64b35d 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -1360,6 +1360,7 @@ async def _save_adopted_session_download(authorize_request_hop=authorize_request_hop,download_scope=download_scope,approved_initial_url=download.url,+ normalize_query_backslashes=True,)body = response.bodyif not body:diff --git a/skyvern/webeye/cdp_download_interceptor.py b/skyvern/webeye/cdp_download_interceptor.pyindex 415c04104..95b07fc9d 100644--- a/skyvern/webeye/cdp_download_interceptor.py+++ b/skyvern/webeye/cdp_download_interceptor.py@@ -1868,6 +1868,7 @@ class CDPDownloadInterceptor:authorize_request_hop=self._redirect_hop_authorizer,download_scope=self._download_scope,approved_initial_url=url,+ normalize_query_backslashes=True,)except Exception as exc:# The download URL is credential-bearing and can reappear inside an exception messagediff --git a/tests/unit/test_adopted_session_download_save.py b/tests/unit/test_adopted_session_download_save.pyindex 19942b340..df43f8287 100644--- a/tests/unit/test_adopted_session_download_save.py+++ b/tests/unit/test_adopted_session_download_save.py@@ -12,8 +12,11 @@ returns the bytes that way.import asyncioimport base64+import threading+import urllib.parsefrom collections.abc import Awaitable, Callablefrom functools import partial+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom pathlib import Pathfrom types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patch@@ -21,6 +24,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom structlog.testing import capture_logs+from skyvern.config import settingsfrom skyvern.forge.sdk.browser_network_egress_monitor import BrowserNetworkEgressMonitorfrom skyvern.forge.sdk.core.http_request_authorization import (RunScopedRedirectHopAuthorizer,@@ -173,6 +177,7 @@ async def test_save_as_raises_target_closed_falls_back_to_refetch(tmp_path) -> Nauthorize_request_hop=_authorize_request_hop,download_scope=None,approved_initial_url=download.url,+ normalize_query_backslashes=True,)page.context.request.get.assert_not_awaited()@@ -198,6 +203,7 @@ async def test_zero_byte_save_as_falls_back_to_refetch(tmp_path) -> None:authorize_request_hop=_authorize_request_hop,download_scope=None,approved_initial_url=download.url,+ normalize_query_backslashes=True,)# the empty placeholder must not survive alongside the recovered fileassert sorted(p.name for p in tmp_path.iterdir()) == [saved.name]@@ -1090,3 +1096,55 @@ async def test_rebind_rotates_download_authority(tmp_path, monkeypatch: pytest.Massert interceptor._redirect_hop_authorizer.download_scope == "next_run"assert interceptor._redirect_hop_authorizer is not prior_authorizerassert context._skyvern_download_run_id == "next_run"+++# --- SKY-15916: adopted-session re-fetch opts into query/fragment backslash normalization (Fix B) ---++_SKY15916_WINQ = r"\V\segment\0\adopted_metered.zip.gpg"+_SKY15916_BODY = b"ADOPTED-SESSION-SYNTHETIC-BYTES\n"+_SKY15916_RECV: list[str] = []+++class _Sky15916Handler(BaseHTTPRequestHandler):+ def log_message(self, *_a: object) -> None:+ pass++ def do_GET(self) -> None:+ _SKY15916_RECV.append(urllib.parse.urlsplit(self.path).query)+ self.send_response(200)+ self.send_header("Content-Type", "application/octet-stream")+ self.send_header("Content-Disposition", 'attachment; filename="adopted.bin"')+ self.send_header("Content-Length", str(len(_SKY15916_BODY)))+ self.end_headers()+ self.wfile.write(_SKY15916_BODY)+++async def test_adopted_session_refetch_saves_query_backslash_via_real_authorizer(tmp_path, monkeypatch) -> None:+ """Fix B / production path: after the eager save_as yields nothing, the adopted-session re-fetch opts+ into query-backslash normalization, so a Windows-query URL saves through the REAL run-scoped+ authorizer (not a permissive stub). Without the opt-in, fetch_file_bytes refuses it and this returns+ None. Synthetic fixture only."""+ _SKY15916_RECV.clear()+ server = ThreadingHTTPServer(("127.0.0.1", 0), _Sky15916Handler)+ port = server.server_address[1]+ threading.Thread(target=server.serve_forever, daemon=True).start()+ monkeypatch.setattr(settings, "ALLOWED_HOSTS", [*settings.ALLOWED_HOSTS, "127.0.0.1"])+ try:+ url = f"http://127.0.0.1:{port}/ELABSTelFileDownload.aspx?TelFile={_SKY15916_WINQ}"+ download = _download(suggested="adopted.bin", url=url) # save_as writes nothing -> falls through to re-fetch+ page = _page_with_refetch()+ saved = await _save_adopted_session_download_impl(+ download,+ page,+ tmp_path,+ authorize_request_hop=RunScopedRedirectHopAuthorizer(download_scope="adopt-scope"),+ request_headers={},+ download_scope="adopt-scope",+ workflow_run_id="wr",+ )+ assert saved is not None and saved.exists()+ assert saved.read_bytes() == _SKY15916_BODY+ assert _SKY15916_RECV and urllib.parse.parse_qs(_SKY15916_RECV[-1])["TelFile"] == [_SKY15916_WINQ]+ finally:+ server.shutdown()diff --git a/tests/unit/test_cdp_download_interceptor.py b/tests/unit/test_cdp_download_interceptor.pyindex d16c98b00..ddb402413 100644--- a/tests/unit/test_cdp_download_interceptor.py+++ b/tests/unit/test_cdp_download_interceptor.py@@ -4188,6 +4188,7 @@ class TestDirectHttpDownloadAuthAndHtmlGuard:authorize_request_hop=authorizer,download_scope=None,approved_initial_url="https://site.example/report.pdf?sig=secret",+ normalize_query_backslashes=True,)assert (tmp_path / "report.pdf").read_bytes() == b"%PDF-1.4 private report"@@ -5652,3 +5653,22 @@ def test_fdopen_failure_closes_raw_fd_without_double_closing_dir_fd(assert len(opened) == 2 # the pinned directory fd and the file fdassert sorted(opened) == sorted(closed) # every opened fd was closed — no leakassert len(closed) == len(set(closed)) # each fd closed exactly once — no double-close+++async def test_download_url_directly_opts_into_query_backslash_normalization(tmp_path: Path) -> None:+ """The CDP download re-fetch opts the guarded seam into query/fragment backslash tolerance, so a+ Windows-path download URL is authorized here while every other canonicalize_origin caller stays strict."""+ interceptor = _make_interceptor(output_dir=str(tmp_path))+ interceptor._browser_context = MagicMock()+ interceptor._cookie_header_for_url = AsyncMock(return_value="")++ guarded_fetch = AsyncMock(side_effect=RuntimeError("stop after capturing the guarded-fetch call"))+ with patch.object(mod.file_api, "fetch_file_bytes", guarded_fetch, create=True):+ await interceptor._download_url_directly(+ "https://host.example/ELABSTelFileDownload.aspx?TelFile=\\V\\segment\\0\\sample.zip.gpg",+ "sample.zip.gpg",+ )++ assert guarded_fetch.await_count == 1+ assert guarded_fetch.await_args.kwargs.get("normalize_query_backslashes") is Truediff --git a/tests/unit/test_copilot_runner_deadline.py b/tests/unit/test_copilot_runner_deadline.pyindex d812375a4..52a9e5683 100644--- a/tests/unit/test_copilot_runner_deadline.py+++ b/tests/unit/test_copilot_runner_deadline.py@@ -536,8 +536,10 @@ async def test_completed_run_survives_an_enrichment_probe_cancelled_by_the_deadlmonkeypatch: pytest.MonkeyPatch,) -> None:cancelled: list[str] = []+ probe_entered = asyncio.Event()async def _hanging(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:+ probe_entered.set()try:await asyncio.Event().wait()except asyncio.CancelledError:@@ -547,11 +549,17 @@ async def test_completed_run_survives_an_enrichment_probe_cancelled_by_the_deadlctx, counts = await _run_with_failing_enrichment(monkeypatch, _hanging)- with pytest.raises(asyncio.TimeoutError):- await asyncio.wait_for(- _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx),- timeout=0.05,- )+ run = asyncio.ensure_future(+ _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)+ )+ try:+ await asyncio.wait_for(probe_entered.wait(), timeout=5)+ with pytest.raises(asyncio.TimeoutError):+ await asyncio.wait_for(run, timeout=0.05)+ finally:+ run.cancel()+ with contextlib.suppress(asyncio.CancelledError):+ await runassert cancelled == ["cancelled"]_assert_later_run_survived(ctx, counts)diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.pyindex 4e0029068..65646a7d6 100644--- a/tests/unit/test_download_file_action_handler.py+++ b/tests/unit/test_download_file_action_handler.py@@ -4760,6 +4760,7 @@ async def test_handle_action_adopted_session_refetches_when_save_as_target_closeauthorize_request_hop=authorize_request_hop,download_scope=None,approved_initial_url=download.url,+ normalize_query_backslashes=True,)page.context._skyvern_cdp_download_interceptor._cookie_header_for_url.assert_awaited_once_with(download.url)page.context.request.get.assert_not_called()@@ -4886,6 +4887,7 @@ async def test_handle_action_adopted_session_falls_through_to_session_folder_wheauthorize_request_hop=authorize_request_hop,download_scope=None,approved_initial_url=download.url,+ normalize_query_backslashes=True,)# storage was polled at least twice: once before the action, again on a later loop iterationassert mock_app.STORAGE.list_downloaded_files_in_browser_session.await_count >= 2@@ -5140,6 +5142,7 @@ async def test_handle_action_adopted_session_xhr_staging_recovered_when_helper_fauthorize_request_hop=authorize_request_hop,download_scope=None,approved_initial_url=download.url,+ normalize_query_backslashes=True,)page.context.request.get.assert_not_called()# file moved from staging to download dirdiff --git a/tests/unit/test_download_query_backslash_refetch_e2e.py b/tests/unit/test_download_query_backslash_refetch_e2e.pynew file mode 100644index 000000000..69aa4bff7--- /dev/null+++ b/tests/unit/test_download_query_backslash_refetch_e2e.py@@ -0,0 +1,125 @@+"""Real-Chromium E2E: the CDP download re-fetch saves a download whose URL carries a Windows-style+path (backslashes) in a query parameter.++Exercises the actual interceptor -> guarded re-fetch (validate+pin SSRF guard, encode_url wire form,+aiohttp transport) -> run-directory persistence chain, against a local HTTP server and a real headless+Chromium context on the origin. Fixtures are synthetic — no customer identifiers, WPIDs, run IDs, or+customer URLs. Mirrors the harness of ``test_download_in_page_fetch_recovery_e2e`` but drives the real+guarded fetch rather than a patched one, so the query-backslash acceptance is proven end to end.+"""++from __future__ import annotations++import threading+import urllib.parse+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from pathlib import Path+from typing import Any+from unittest.mock import MagicMock++import pytest+import pytest_asyncio++import skyvern.webeye.cdp_download_interceptor as mod+from skyvern.config import settings+from skyvern.forge.sdk.core.http_request_authorization import RunScopedRedirectHopAuthorizer++# A Windows/UNC-style path in a query parameter, the shape ASP.NET download endpoints emit. Synthetic.+_WINDOWS_QUERY = r"\V\segment\0\sample_metered.zip.gpg"+_SAMPLE_BODY = b"SKYVERN-SYNTHETIC-DOWNLOAD-BYTES\n"+_RECEIVED_QUERY: list[str] = []+++def _deps_available() -> bool:+ try:+ from playwright.sync_api import sync_playwright++ with sync_playwright() as p:+ return Path(p.chromium.executable_path).exists()+ except Exception:+ return False+++pytestmark = pytest.mark.skipif(+ not _deps_available(), reason="Requires Playwright chromium (playwright install chromium)"+)+++class _Handler(BaseHTTPRequestHandler):+ def log_message(self, *_args: Any) -> None:+ pass++ def do_GET(self) -> None:+ parsed = urllib.parse.urlsplit(self.path)+ if parsed.path == "/":+ body = b"<!doctype html><html><body>ready</body></html>"+ self.send_response(200)+ self.send_header("Content-Type", "text/html; charset=utf-8")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)+ return+ _RECEIVED_QUERY.append(parsed.query)+ self.send_response(200)+ self.send_header("Content-Type", "application/octet-stream")+ self.send_header("Content-Disposition", 'attachment; filename="sample.bin"')+ self.send_header("Content-Length", str(len(_SAMPLE_BODY)))+ self.end_headers()+ self.wfile.write(_SAMPLE_BODY)+++@pytest_asyncio.fixture+async def http_site(monkeypatch: pytest.MonkeyPatch) -> Any:+ _RECEIVED_QUERY.clear()+ server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)+ port = server.server_address[1]+ threading.Thread(target=server.serve_forever, daemon=True).start()+ monkeypatch.setattr(settings, "ALLOWED_HOSTS", [*settings.ALLOWED_HOSTS, "127.0.0.1"])++ from playwright.async_api import async_playwright++ async with async_playwright() as p:+ browser = await p.chromium.launch(+ headless=True,+ args=["--no-proxy-server", "--proxy-bypass-list=*"],+ proxy={"server": "direct://"},+ )+ context = await browser.new_context()+ page = await context.new_page()+ await page.goto(f"http://127.0.0.1:{port}/", wait_until="domcontentloaded")+ try:+ yield context, page, port+ finally:+ await browser.close()+ server.shutdown()+++def _make_interceptor(output_dir: str) -> mod.CDPDownloadInterceptor:+ monitor = MagicMock()+ monitor.authorize_request.return_value = True+ return mod.CDPDownloadInterceptor(+ output_dir=output_dir,+ network_egress_monitor=monitor,+ redirect_hop_authorizer=RunScopedRedirectHopAuthorizer(download_scope="e2e-scope"),+ )+++async def test_cdp_refetch_saves_query_backslash_download(http_site: Any, tmp_path: Path) -> None:+ context, page, port = http_site+ url = f"http://127.0.0.1:{port}/ELABSTelFileDownload.aspx?TelFile={_WINDOWS_QUERY}"++ interceptor = _make_interceptor(str(tmp_path))+ interceptor._browser_context = context+ await interceptor.enable_for_page(page)+ try:+ await interceptor._download_url_directly(url, "sample.bin")+ finally:+ await interceptor.disable()++ saved = list(tmp_path.iterdir())+ assert len(saved) == 1, "the query-backslash download was not saved through the guarded re-fetch"+ assert saved[0].read_bytes() == _SAMPLE_BODY+ # The server received the identical Windows path the browser reported (percent-encoded on the wire).+ assert _RECEIVED_QUERY, "the guarded re-fetch never reached the server"+ assert urllib.parse.parse_qs(_RECEIVED_QUERY[-1])["TelFile"] == [_WINDOWS_QUERY]diff --git a/tests/unit/test_fetch_file_bytes_query_backslash.py b/tests/unit/test_fetch_file_bytes_query_backslash.pynew file mode 100644index 000000000..d9a1c796d--- /dev/null+++ b/tests/unit/test_fetch_file_bytes_query_backslash.py@@ -0,0 +1,218 @@+"""The guarded download re-fetch accepts benign Windows-path backslashes in the query/fragment.++A ``file_download`` whose href carries a Windows-style path in a query parameter (backslashes) was+refused before any network request: the origin precondition in the guarded re-fetch rejects any URL+containing a backslash, and with the browser's own download denied there is no fallback, so the file+is never saved.++These tests drive the REAL production authorizer (``RunScopedRedirectHopAuthorizer``) — NOT a+permissive stub — because that authorizer's first-hop binding independently canonicalizes the target+(``canonicalize_effect_target`` → ``canonicalize_origin``) and would otherwise reject the same+backslash URL after the precheck. They exercise the real chain (validate+pin SSRF guard, the run-scoped+authorization/binding, ``encode_url`` wire serialization, aiohttp transport) against a local server.+Fixtures are synthetic — no customer identifiers, WPIDs, run IDs, or customer URLs.+"""++from __future__ import annotations++import threading+import urllib.parse+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from typing import Any++import pytest++from skyvern.config import settings+from skyvern.forge.sdk.api import files as files_mod+from skyvern.forge.sdk.browser_action_policy import canonicalize_origin+from skyvern.forge.sdk.core.http_request_authorization import RunScopedRedirectHopAuthorizer++HttpException = files_mod.HttpException+fetch_file_bytes = files_mod.fetch_file_bytes+_origin_authorizable_url = files_mod._origin_authorizable_url++# A Windows/UNC-style path in a query parameter, the shape ASP.NET download endpoints emit. Synthetic.+_WINDOWS_QUERY = r"\V\segment\0\sample_metered.zip.gpg"+_ENCODED_QUERY = "%5CV%5Csegment%5C0%5Csample_metered.zip.gpg"+_SAMPLE_BODY = b"SKYVERN-SYNTHETIC-DOWNLOAD-BYTES\n"+_SCOPE = "run-scope-1"+++class _RecordingHandler(BaseHTTPRequestHandler):+ """Serve a small non-empty body; record the raw query; 302 to a denied origin on /redirect-denied."""++ received_raw_query: list[str] = []++ def log_message(self, *_args: Any) -> None:+ pass++ def do_GET(self) -> None:+ parsed = urllib.parse.urlsplit(self.path)+ if parsed.path == "/redirect-denied":+ self.send_response(302)+ self.send_header("Location", "http://169.254.169.254/blocked")+ self.end_headers()+ return+ type(self).received_raw_query.append(parsed.query)+ self.send_response(200)+ self.send_header("Content-Type", "application/octet-stream")+ self.send_header("Content-Disposition", 'attachment; filename="sample.bin"')+ self.send_header("Content-Length", str(len(_SAMPLE_BODY)))+ self.end_headers()+ self.wfile.write(_SAMPLE_BODY)+++def local_server(monkeypatch: pytest.MonkeyPatch) -> Any:+ _RecordingHandler.received_raw_query = []+ server = ThreadingHTTPServer(("127.0.0.1", 0), _RecordingHandler)+ port = server.server_address[1]+ threading.Thread(target=server.serve_forever, daemon=True).start()+ # The SSRF guard blocks loopback by default; the local fixture is explicitly allowed, exactly as+ # the existing adopted-session download integration test does.+ monkeypatch.setattr(settings, "ALLOWED_HOSTS", [*settings.ALLOWED_HOSTS, "127.0.0.1"])+ try:+ yield f"http://127.0.0.1:{port}", _RecordingHandler+ finally:+ server.shutdown()+++def _authz() -> RunScopedRedirectHopAuthorizer:+ return RunScopedRedirectHopAuthorizer(download_scope=_SCOPE)+++# --- pure helper: what the origin check / authorizer is allowed to see -------------------------+++def test_origin_authorizable_url_encodes_only_query_and_fragment() -> None:+ raw = "https://host.example/ELABSTelFileDownload.aspx?TelFile=" + _WINDOWS_QUERY+ normalized = _origin_authorizable_url(raw)+ assert canonicalize_origin(raw) is None+ assert "\\" not in normalized+ assert canonicalize_origin(normalized) is not None+ assert canonicalize_origin(_origin_authorizable_url("https://host.example/p#a\\b")) is not None+++ "raw",+ [+ "https://good.example\\@evil.example/f?x=1",+ "https:/\\good.example/f?x=1",+ "https://good.example\\.evil.example/?x=1",+ "https://host.example/a\\b?x=1",+ "https://host.example/a\\b",+ ],+)+def test_origin_authorizable_url_keeps_scheme_authority_path_fail_closed(raw: str) -> None:+ assert canonicalize_origin(_origin_authorizable_url(raw)) is None+++def test_origin_authorizable_url_leaves_ordinary_urls_untouched() -> None:+ for url in ("https://host.example/path", "https://host.example/p?x=1&y=2", "https://host.example/"):+ assert _origin_authorizable_url(url) == url+++# --- real production authorizer + real re-fetch chain ------------------------------------------+++async def test_production_authorizer_saves_benign_query_backslash(local_server: Any) -> None:+ base, handler = local_server+ url = f"{base}/ELABSTelFileDownload.aspx?TelFile={_WINDOWS_QUERY}"+ response = await fetch_file_bytes(+ url,+ authorize_request_hop=_authz(),+ download_scope=_SCOPE,+ approved_initial_url=url,+ normalize_query_backslashes=True,+ )+ assert response.body == _SAMPLE_BODY and response.filename == "sample.bin"+ # The run-scoped authorizer admitted the hop AND the server received the exact Windows path.+ assert handler.received_raw_query, "the guarded re-fetch never reached the server"+ assert urllib.parse.parse_qs(handler.received_raw_query[-1])["TelFile"] == [_WINDOWS_QUERY]+++async def test_production_authorizer_saves_already_encoded_query(local_server: Any) -> None:+ base, handler = local_server+ url = f"{base}/dl.aspx?TelFile={_ENCODED_QUERY}"+ response = await fetch_file_bytes(+ url,+ authorize_request_hop=_authz(),+ download_scope=_SCOPE,+ approved_initial_url=url,+ normalize_query_backslashes=True,+ )+ assert response.body == _SAMPLE_BODY+ # Already-encoded %5C is not double-encoded; the server decodes it back to the backslash path.+ assert urllib.parse.parse_qs(handler.received_raw_query[-1])["TelFile"] == [_WINDOWS_QUERY]+++async def test_default_off_rejected_at_precheck(local_server: Any) -> None:+ base, handler = local_server+ url = f"{base}/dl.aspx?TelFile={_WINDOWS_QUERY}"+ # Default (opt-in off): the precheck refuses before any network request or authorizer call.+ with pytest.raises(HttpException, match="browser-canonicalizable"):+ await fetch_file_bytes(url, authorize_request_hop=_authz(), download_scope=_SCOPE, approved_initial_url=url)+ assert handler.received_raw_query == []+++async def test_wrong_first_target_binding_rejected(local_server: Any) -> None:+ base, handler = local_server+ port = int(urllib.parse.urlsplit(base).port)+ url = f"{base}/dl.aspx?TelFile={_WINDOWS_QUERY}"+ # approved_initial_url has a DIFFERENT origin (port); the run-scoped first-hop binding must refuse,+ # proving the opt-in did not weaken the target/initial origin binding.+ other_initial = f"http://127.0.0.1:{port + 1}/dl.aspx?TelFile={_WINDOWS_QUERY}"+ with pytest.raises(PermissionError):+ await fetch_file_bytes(+ url,+ authorize_request_hop=_authz(),+ download_scope=_SCOPE,+ approved_initial_url=other_initial,+ normalize_query_backslashes=True,+ )+ assert handler.received_raw_query == []+++async def test_disallowed_redirect_fails_closed(local_server: Any) -> None:+ base, _ = local_server+ url = f"{base}/redirect-denied?TelFile={_WINDOWS_QUERY}"+ # First hop is admitted (same origin) and reaches the server, which 302s to a link-local metadata+ # address; the redirect hop is still validated and pinned, so the denied origin fails closed.+ with pytest.raises(Exception) as excinfo:+ await fetch_file_bytes(+ url,+ authorize_request_hop=_authz(),+ download_scope=_SCOPE,+ approved_initial_url=url,+ normalize_query_backslashes=True,+ )+ assert type(excinfo.value).__name__ in {"BlockedHost", "SkyvernHTTPException", "HttpException"}+++async def test_ssrf_target_refused_with_opt_in() -> None:+ # A link-local metadata address is refused by the host guard before the authorizer, even opted in.+ with pytest.raises(Exception) as excinfo:+ await fetch_file_bytes(+ "http://169.254.169.254/latest/meta-data?p=\\x\\y",+ authorize_request_hop=_authz(),+ download_scope=_SCOPE,+ approved_initial_url="http://169.254.169.254/latest/meta-data?p=\\x\\y",+ normalize_query_backslashes=True,+ )+ assert type(excinfo.value).__name__ in {"BlockedHost", "SkyvernHTTPException", "HttpException"}+++async def test_ordinary_no_backslash_default_saves(local_server: Any) -> None:+ base, _ = local_server+ url = f"{base}/dl.aspx?x=1"+ response = await fetch_file_bytes(+ url, authorize_request_hop=_authz(), download_scope=_SCOPE, approved_initial_url=url+ )+ assert response.body == _SAMPLE_BODY