Skip to content

Automate browser based workflows with AI

Admit a blog route removal only behind a reviewed 301 redirect (SKY-15847) (#8496)

Shuchang Zheng46fc8f0d817f
diff --git a/skyvern/forge/sdk/api/files.py b/skyvern/forge/sdk/api/files.py
index aa37a0a29..34b55335e 100644
--- a/skyvern/forge/sdk/api/files.py
+++ b/skyvern/forge/sdk/api/files.py
@@ -248,6 +248,22 @@ class GuardedFileResponse:
GuardedFileFetchHopResult: TypeAlias = GuardedFileRedirect | GuardedFileResponse
+def _origin_authorizable_url(url: str) -> str:
+ """Return *url* with query/fragment backslashes percent-encoded so the origin guard does not
+ refuse a benign Windows-style path carried there. Scheme, authority, and path keep their
+ backslashes so this stays fail-closed on an authority-parse divergence; at the download seam that
+ is defense-in-depth, because validate_and_pin (AnyHttpUrl/WHATWG) already normalizes authority and
+ path backslashes before the origin check. Only for the origin check — never the fetched URL."""
+ cut = len(url)
+ for separator in ("?", "#"):
+ index = url.find(separator)
+ if index != -1 and index < cut:
+ cut = index
+ if cut == len(url):
+ return url
+ return url[:cut] + url[cut:].replace("\\", "%5C")
+
+
async def fetch_file_bytes(
url: str,
*,
@@ -258,6 +274,7 @@ async def fetch_file_bytes(
authorize_request_hop: RedirectHopAuthorizer[GuardedFileFetchHopResult],
download_scope: str | None = None,
approved_initial_url: str | None = None,
+ normalize_query_backslashes: bool = False,
) -> GuardedFileResponse:
"""Fetch a bounded HTTP file through the validated, pinned, per-hop authorization seam.
@@ -272,17 +289,28 @@ async def fetch_file_bytes(
resolver = SSRFGuardedResolver()
current_url = await validate_and_pin_fetch_url(url, resolver)
- if canonicalize_origin(current_url) is None:
+ origin_source = _origin_authorizable_url(current_url) if normalize_query_backslashes else current_url
+ if canonicalize_origin(origin_source) is None:
raise HttpException(400, "[redacted]", "URL has no browser-canonicalizable HTTP origin")
if allowed_redirect_origin is not None and _url_origin(current_url) != _url_origin(allowed_redirect_origin):
raise HttpException(400, "[redacted]", "Cross-origin redirect blocked by policy")
+ # The run-scoped authorizer canonicalizes target_url/initial_url via canonicalize_effect_target,
+ # which refuses a backslash exactly like the precheck above. Present it the same query-normalized
+ # form; current_url below stays raw so the encoded wire request and SSRF pinning are unchanged.
+ authz_initial_url = (
+ _origin_authorizable_url(approved_initial_url)
+ if normalize_query_backslashes and approved_initial_url is not None
+ else approved_initial_url
+ )
+
request_headers = dict(headers or {})
source_url: str | None = None
max_size_bytes = max_size_mb * 1024 * 1024
async with aiohttp.ClientSession(connector=ssrf_guarded_tcp_connector(resolver)) as session:
for _ in range(MAX_SAFE_REDIRECTS + 1):
encoded_url = encode_url(current_url)
+ authz_target_url = _origin_authorizable_url(current_url) if normalize_query_backslashes else current_url
async def dispatch(_resolved_values: tuple[str, ...]) -> GuardedFileFetchHopResult:
async with session.get(
@@ -312,10 +340,10 @@ async def fetch_file_bytes(
authorize_request_hop,
RedirectHopAuthorization(
source_url=source_url,
- target_url=current_url,
+ target_url=authz_target_url,
method="GET",
download_scope=download_scope,
- initial_url=approved_initial_url,
+ initial_url=authz_initial_url,
),
dispatch,
)
@@ -323,7 +351,8 @@ async def fetch_file_bytes(
return result
next_url = await validate_and_pin_redirect_url(current_url, result.location, resolver)
- if canonicalize_origin(next_url) is None:
+ next_origin_source = _origin_authorizable_url(next_url) if normalize_query_backslashes else next_url
+ if canonicalize_origin(next_origin_source) is None:
raise HttpException(400, "[redacted]", "Redirect has no browser-canonicalizable HTTP origin")
if allowed_redirect_origin is not None and _url_origin(next_url) != _url_origin(allowed_redirect_origin):
raise HttpException(400, "[redacted]", "Cross-origin redirect blocked by policy")
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 99d70afd7..aea64b35d 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -1360,6 +1360,7 @@ async def _save_adopted_session_download(
authorize_request_hop=authorize_request_hop,
download_scope=download_scope,
approved_initial_url=download.url,
+ normalize_query_backslashes=True,
)
body = response.body
if not body:
diff --git a/skyvern/webeye/cdp_download_interceptor.py b/skyvern/webeye/cdp_download_interceptor.py
index 415c04104..95b07fc9d 100644
--- a/skyvern/webeye/cdp_download_interceptor.py
+++ b/skyvern/webeye/cdp_download_interceptor.py
@@ -1868,6 +1868,7 @@ class CDPDownloadInterceptor:
authorize_request_hop=self._redirect_hop_authorizer,
download_scope=self._download_scope,
approved_initial_url=url,
+ normalize_query_backslashes=True,
)
except Exception as exc:
# The download URL is credential-bearing and can reappear inside an exception message
diff --git a/tests/unit/test_adopted_session_download_save.py b/tests/unit/test_adopted_session_download_save.py
index 19942b340..df43f8287 100644
--- a/tests/unit/test_adopted_session_download_save.py
+++ b/tests/unit/test_adopted_session_download_save.py
@@ -12,8 +12,11 @@ returns the bytes that way.
import asyncio
import base64
+import threading
+import urllib.parse
from collections.abc import Awaitable, Callable
from functools import partial
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
@@ -21,6 +24,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from structlog.testing import capture_logs
+from skyvern.config import settings
from skyvern.forge.sdk.browser_network_egress_monitor import BrowserNetworkEgressMonitor
from skyvern.forge.sdk.core.http_request_authorization import (
RunScopedRedirectHopAuthorizer,
@@ -173,6 +177,7 @@ async def test_save_as_raises_target_closed_falls_back_to_refetch(tmp_path) -> N
authorize_request_hop=_authorize_request_hop,
download_scope=None,
approved_initial_url=download.url,
+ normalize_query_backslashes=True,
)
page.context.request.get.assert_not_awaited()
@@ -198,6 +203,7 @@ async def test_zero_byte_save_as_falls_back_to_refetch(tmp_path) -> None:
authorize_request_hop=_authorize_request_hop,
download_scope=None,
approved_initial_url=download.url,
+ normalize_query_backslashes=True,
)
# the empty placeholder must not survive alongside the recovered file
assert sorted(p.name for p in tmp_path.iterdir()) == [saved.name]
@@ -1090,3 +1096,55 @@ async def test_rebind_rotates_download_authority(tmp_path, monkeypatch: pytest.M
assert interceptor._redirect_hop_authorizer.download_scope == "next_run"
assert interceptor._redirect_hop_authorizer is not prior_authorizer
assert context._skyvern_download_run_id == "next_run"
+
+
+# --- SKY-15916: adopted-session re-fetch opts into query/fragment backslash normalization (Fix B) ---
+
+_SKY15916_WINQ = r"\V\segment\0\adopted_metered.zip.gpg"
+_SKY15916_BODY = b"ADOPTED-SESSION-SYNTHETIC-BYTES\n"
+_SKY15916_RECV: list[str] = []
+
+
+class _Sky15916Handler(BaseHTTPRequestHandler):
+ def log_message(self, *_a: object) -> None:
+ pass
+
+ def do_GET(self) -> None:
+ _SKY15916_RECV.append(urllib.parse.urlsplit(self.path).query)
+ self.send_response(200)
+ self.send_header("Content-Type", "application/octet-stream")
+ self.send_header("Content-Disposition", 'attachment; filename="adopted.bin"')
+ self.send_header("Content-Length", str(len(_SKY15916_BODY)))
+ self.end_headers()
+ self.wfile.write(_SKY15916_BODY)
+
+
+async def test_adopted_session_refetch_saves_query_backslash_via_real_authorizer(tmp_path, monkeypatch) -> None:
+ """Fix B / production path: after the eager save_as yields nothing, the adopted-session re-fetch opts
+ into query-backslash normalization, so a Windows-query URL saves through the REAL run-scoped
+ authorizer (not a permissive stub). Without the opt-in, fetch_file_bytes refuses it and this returns
+ None. Synthetic fixture only."""
+ _SKY15916_RECV.clear()
+ server = ThreadingHTTPServer(("127.0.0.1", 0), _Sky15916Handler)
+ port = server.server_address[1]
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ monkeypatch.setattr(settings, "ALLOWED_HOSTS", [*settings.ALLOWED_HOSTS, "127.0.0.1"])
+ try:
+ url = f"http://127.0.0.1:{port}/ELABSTelFileDownload.aspx?TelFile={_SKY15916_WINQ}"
+ download = _download(suggested="adopted.bin", url=url) # save_as writes nothing -> falls through to re-fetch
+ page = _page_with_refetch()
+ saved = await _save_adopted_session_download_impl(
+ download,
+ page,
+ tmp_path,
+ authorize_request_hop=RunScopedRedirectHopAuthorizer(download_scope="adopt-scope"),
+ request_headers={},
+ download_scope="adopt-scope",
+ workflow_run_id="wr",
+ )
+ assert saved is not None and saved.exists()
+ assert saved.read_bytes() == _SKY15916_BODY
+ assert _SKY15916_RECV and urllib.parse.parse_qs(_SKY15916_RECV[-1])["TelFile"] == [_SKY15916_WINQ]
+ finally:
+ server.shutdown()
diff --git a/tests/unit/test_cdp_download_interceptor.py b/tests/unit/test_cdp_download_interceptor.py
index d16c98b00..ddb402413 100644
--- a/tests/unit/test_cdp_download_interceptor.py
+++ b/tests/unit/test_cdp_download_interceptor.py
@@ -4188,6 +4188,7 @@ class TestDirectHttpDownloadAuthAndHtmlGuard:
authorize_request_hop=authorizer,
download_scope=None,
approved_initial_url="https://site.example/report.pdf?sig=secret",
+ normalize_query_backslashes=True,
)
assert (tmp_path / "report.pdf").read_bytes() == b"%PDF-1.4 private report"
@@ -5652,3 +5653,22 @@ def test_fdopen_failure_closes_raw_fd_without_double_closing_dir_fd(
assert len(opened) == 2 # the pinned directory fd and the file fd
assert sorted(opened) == sorted(closed) # every opened fd was closed — no leak
assert len(closed) == len(set(closed)) # each fd closed exactly once — no double-close
+
+
+async def test_download_url_directly_opts_into_query_backslash_normalization(tmp_path: Path) -> None:
+ """The CDP download re-fetch opts the guarded seam into query/fragment backslash tolerance, so a
+ Windows-path download URL is authorized here while every other canonicalize_origin caller stays strict."""
+ interceptor = _make_interceptor(output_dir=str(tmp_path))
+ interceptor._browser_context = MagicMock()
+ interceptor._cookie_header_for_url = AsyncMock(return_value="")
+
+ guarded_fetch = AsyncMock(side_effect=RuntimeError("stop after capturing the guarded-fetch call"))
+ with patch.object(mod.file_api, "fetch_file_bytes", guarded_fetch, create=True):
+ await interceptor._download_url_directly(
+ "https://host.example/ELABSTelFileDownload.aspx?TelFile=\\V\\segment\\0\\sample.zip.gpg",
+ "sample.zip.gpg",
+ )
+
+ assert guarded_fetch.await_count == 1
+ assert guarded_fetch.await_args.kwargs.get("normalize_query_backslashes") is True
diff --git a/tests/unit/test_copilot_runner_deadline.py b/tests/unit/test_copilot_runner_deadline.py
index d812375a4..52a9e5683 100644
--- a/tests/unit/test_copilot_runner_deadline.py
+++ b/tests/unit/test_copilot_runner_deadline.py
@@ -536,8 +536,10 @@ async def test_completed_run_survives_an_enrichment_probe_cancelled_by_the_deadl
monkeypatch: pytest.MonkeyPatch,
) -> None:
cancelled: list[str] = []
+ probe_entered = asyncio.Event()
async def _hanging(*_args: object, **_kwargs: object) -> tuple[str, dict[str, object] | None]:
+ probe_entered.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
@@ -547,11 +549,17 @@ async def test_completed_run_survives_an_enrichment_probe_cancelled_by_the_deadl
ctx, counts = await _run_with_failing_enrichment(monkeypatch, _hanging)
- with pytest.raises(asyncio.TimeoutError):
- await asyncio.wait_for(
- _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx),
- timeout=0.05,
- )
+ run = asyncio.ensure_future(
+ _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+ )
+ try:
+ await asyncio.wait_for(probe_entered.wait(), timeout=5)
+ with pytest.raises(asyncio.TimeoutError):
+ await asyncio.wait_for(run, timeout=0.05)
+ finally:
+ run.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await run
assert cancelled == ["cancelled"]
_assert_later_run_survived(ctx, counts)
diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.py
index 4e0029068..65646a7d6 100644
--- a/tests/unit/test_download_file_action_handler.py
+++ b/tests/unit/test_download_file_action_handler.py
@@ -4760,6 +4760,7 @@ async def test_handle_action_adopted_session_refetches_when_save_as_target_close
authorize_request_hop=authorize_request_hop,
download_scope=None,
approved_initial_url=download.url,
+ normalize_query_backslashes=True,
)
page.context._skyvern_cdp_download_interceptor._cookie_header_for_url.assert_awaited_once_with(download.url)
page.context.request.get.assert_not_called()
@@ -4886,6 +4887,7 @@ async def test_handle_action_adopted_session_falls_through_to_session_folder_whe
authorize_request_hop=authorize_request_hop,
download_scope=None,
approved_initial_url=download.url,
+ normalize_query_backslashes=True,
)
# storage was polled at least twice: once before the action, again on a later loop iteration
assert mock_app.STORAGE.list_downloaded_files_in_browser_session.await_count >= 2
@@ -5140,6 +5142,7 @@ async def test_handle_action_adopted_session_xhr_staging_recovered_when_helper_f
authorize_request_hop=authorize_request_hop,
download_scope=None,
approved_initial_url=download.url,
+ normalize_query_backslashes=True,
)
page.context.request.get.assert_not_called()
# file moved from staging to download dir
diff --git a/tests/unit/test_download_query_backslash_refetch_e2e.py b/tests/unit/test_download_query_backslash_refetch_e2e.py
new file mode 100644
index 000000000..69aa4bff7
--- /dev/null
+++ b/tests/unit/test_download_query_backslash_refetch_e2e.py
@@ -0,0 +1,125 @@
+"""Real-Chromium E2E: the CDP download re-fetch saves a download whose URL carries a Windows-style
+path (backslashes) in a query parameter.
+
+Exercises the actual interceptor -> guarded re-fetch (validate+pin SSRF guard, encode_url wire form,
+aiohttp transport) -> run-directory persistence chain, against a local HTTP server and a real headless
+Chromium context on the origin. Fixtures are synthetic — no customer identifiers, WPIDs, run IDs, or
+customer URLs. Mirrors the harness of ``test_download_in_page_fetch_recovery_e2e`` but drives the real
+guarded fetch rather than a patched one, so the query-backslash acceptance is proven end to end.
+"""
+
+from __future__ import annotations
+
+import threading
+import urllib.parse
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+import pytest_asyncio
+
+import skyvern.webeye.cdp_download_interceptor as mod
+from skyvern.config import settings
+from skyvern.forge.sdk.core.http_request_authorization import RunScopedRedirectHopAuthorizer
+
+# A Windows/UNC-style path in a query parameter, the shape ASP.NET download endpoints emit. Synthetic.
+_WINDOWS_QUERY = r"\V\segment\0\sample_metered.zip.gpg"
+_SAMPLE_BODY = b"SKYVERN-SYNTHETIC-DOWNLOAD-BYTES\n"
+_RECEIVED_QUERY: list[str] = []
+
+
+def _deps_available() -> bool:
+ try:
+ from playwright.sync_api import sync_playwright
+
+ with sync_playwright() as p:
+ return Path(p.chromium.executable_path).exists()
+ except Exception:
+ return False
+
+
+pytestmark = pytest.mark.skipif(
+ not _deps_available(), reason="Requires Playwright chromium (playwright install chromium)"
+)
+
+
+class _Handler(BaseHTTPRequestHandler):
+ def log_message(self, *_args: Any) -> None:
+ pass
+
+ def do_GET(self) -> None:
+ parsed = urllib.parse.urlsplit(self.path)
+ if parsed.path == "/":
+ body = b"<!doctype html><html><body>ready</body></html>"
+ self.send_response(200)
+ self.send_header("Content-Type", "text/html; charset=utf-8")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+ return
+ _RECEIVED_QUERY.append(parsed.query)
+ self.send_response(200)
+ self.send_header("Content-Type", "application/octet-stream")
+ self.send_header("Content-Disposition", 'attachment; filename="sample.bin"')
+ self.send_header("Content-Length", str(len(_SAMPLE_BODY)))
+ self.end_headers()
+ self.wfile.write(_SAMPLE_BODY)
+
+
+@pytest_asyncio.fixture
+async def http_site(monkeypatch: pytest.MonkeyPatch) -> Any:
+ _RECEIVED_QUERY.clear()
+ server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
+ port = server.server_address[1]
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ monkeypatch.setattr(settings, "ALLOWED_HOSTS", [*settings.ALLOWED_HOSTS, "127.0.0.1"])
+
+ from playwright.async_api import async_playwright
+
+ async with async_playwright() as p:
+ browser = await p.chromium.launch(
+ headless=True,
+ args=["--no-proxy-server", "--proxy-bypass-list=*"],
+ proxy={"server": "direct://"},
+ )
+ context = await browser.new_context()
+ page = await context.new_page()
+ await page.goto(f"http://127.0.0.1:{port}/", wait_until="domcontentloaded")
+ try:
+ yield context, page, port
+ finally:
+ await browser.close()
+ server.shutdown()
+
+
+def _make_interceptor(output_dir: str) -> mod.CDPDownloadInterceptor:
+ monitor = MagicMock()
+ monitor.authorize_request.return_value = True
+ return mod.CDPDownloadInterceptor(
+ output_dir=output_dir,
+ network_egress_monitor=monitor,
+ redirect_hop_authorizer=RunScopedRedirectHopAuthorizer(download_scope="e2e-scope"),
+ )
+
+
+async def test_cdp_refetch_saves_query_backslash_download(http_site: Any, tmp_path: Path) -> None:
+ context, page, port = http_site
+ url = f"http://127.0.0.1:{port}/ELABSTelFileDownload.aspx?TelFile={_WINDOWS_QUERY}"
+
+ interceptor = _make_interceptor(str(tmp_path))
+ interceptor._browser_context = context
+ await interceptor.enable_for_page(page)
+ try:
+ await interceptor._download_url_directly(url, "sample.bin")
+ finally:
+ await interceptor.disable()
+
+ saved = list(tmp_path.iterdir())
+ assert len(saved) == 1, "the query-backslash download was not saved through the guarded re-fetch"
+ assert saved[0].read_bytes() == _SAMPLE_BODY
+ # The server received the identical Windows path the browser reported (percent-encoded on the wire).
+ assert _RECEIVED_QUERY, "the guarded re-fetch never reached the server"
+ assert urllib.parse.parse_qs(_RECEIVED_QUERY[-1])["TelFile"] == [_WINDOWS_QUERY]
diff --git a/tests/unit/test_fetch_file_bytes_query_backslash.py b/tests/unit/test_fetch_file_bytes_query_backslash.py
new file mode 100644
index 000000000..d9a1c796d
--- /dev/null
+++ b/tests/unit/test_fetch_file_bytes_query_backslash.py
@@ -0,0 +1,218 @@
+"""The guarded download re-fetch accepts benign Windows-path backslashes in the query/fragment.
+
+A ``file_download`` whose href carries a Windows-style path in a query parameter (backslashes) was
+refused before any network request: the origin precondition in the guarded re-fetch rejects any URL
+containing a backslash, and with the browser's own download denied there is no fallback, so the file
+is never saved.
+
+These tests drive the REAL production authorizer (``RunScopedRedirectHopAuthorizer``) — NOT a
+permissive stub — because that authorizer's first-hop binding independently canonicalizes the target
+(``canonicalize_effect_target`` → ``canonicalize_origin``) and would otherwise reject the same
+backslash URL after the precheck. They exercise the real chain (validate+pin SSRF guard, the run-scoped
+authorization/binding, ``encode_url`` wire serialization, aiohttp transport) against a local server.
+Fixtures are synthetic — no customer identifiers, WPIDs, run IDs, or customer URLs.
+"""
+
+from __future__ import annotations
+
+import threading
+import urllib.parse
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Any
+
+import pytest
+
+from skyvern.config import settings
+from skyvern.forge.sdk.api import files as files_mod
+from skyvern.forge.sdk.browser_action_policy import canonicalize_origin
+from skyvern.forge.sdk.core.http_request_authorization import RunScopedRedirectHopAuthorizer
+
+HttpException = files_mod.HttpException
+fetch_file_bytes = files_mod.fetch_file_bytes
+_origin_authorizable_url = files_mod._origin_authorizable_url
+
+# A Windows/UNC-style path in a query parameter, the shape ASP.NET download endpoints emit. Synthetic.
+_WINDOWS_QUERY = r"\V\segment\0\sample_metered.zip.gpg"
+_ENCODED_QUERY = "%5CV%5Csegment%5C0%5Csample_metered.zip.gpg"
+_SAMPLE_BODY = b"SKYVERN-SYNTHETIC-DOWNLOAD-BYTES\n"
+_SCOPE = "run-scope-1"
+
+
+class _RecordingHandler(BaseHTTPRequestHandler):
+ """Serve a small non-empty body; record the raw query; 302 to a denied origin on /redirect-denied."""
+
+ received_raw_query: list[str] = []
+
+ def log_message(self, *_args: Any) -> None:
+ pass
+
+ def do_GET(self) -> None:
+ parsed = urllib.parse.urlsplit(self.path)
+ if parsed.path == "/redirect-denied":
+ self.send_response(302)
+ self.send_header("Location", "http://169.254.169.254/blocked")
+ self.end_headers()
+ return
+ type(self).received_raw_query.append(parsed.query)
+ self.send_response(200)
+ self.send_header("Content-Type", "application/octet-stream")
+ self.send_header("Content-Disposition", 'attachment; filename="sample.bin"')
+ self.send_header("Content-Length", str(len(_SAMPLE_BODY)))
+ self.end_headers()
+ self.wfile.write(_SAMPLE_BODY)
+
+
+def local_server(monkeypatch: pytest.MonkeyPatch) -> Any:
+ _RecordingHandler.received_raw_query = []
+ server = ThreadingHTTPServer(("127.0.0.1", 0), _RecordingHandler)
+ port = server.server_address[1]
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ # The SSRF guard blocks loopback by default; the local fixture is explicitly allowed, exactly as
+ # the existing adopted-session download integration test does.
+ monkeypatch.setattr(settings, "ALLOWED_HOSTS", [*settings.ALLOWED_HOSTS, "127.0.0.1"])
+ try:
+ yield f"http://127.0.0.1:{port}", _RecordingHandler
+ finally:
+ server.shutdown()
+
+
+def _authz() -> RunScopedRedirectHopAuthorizer:
+ return RunScopedRedirectHopAuthorizer(download_scope=_SCOPE)
+
+
+# --- pure helper: what the origin check / authorizer is allowed to see -------------------------
+
+
+def test_origin_authorizable_url_encodes_only_query_and_fragment() -> None:
+ raw = "https://host.example/ELABSTelFileDownload.aspx?TelFile=" + _WINDOWS_QUERY
+ normalized = _origin_authorizable_url(raw)
+ assert canonicalize_origin(raw) is None
+ assert "\\" not in normalized
+ assert canonicalize_origin(normalized) is not None
+ assert canonicalize_origin(_origin_authorizable_url("https://host.example/p#a\\b")) is not None
+
+
+ "raw",
+ [
+ "https://good.example\\@evil.example/f?x=1",
+ "https:/\\good.example/f?x=1",
+ "https://good.example\\.evil.example/?x=1",
+ "https://host.example/a\\b?x=1",
+ "https://host.example/a\\b",
+ ],
+)
+def test_origin_authorizable_url_keeps_scheme_authority_path_fail_closed(raw: str) -> None:
+ assert canonicalize_origin(_origin_authorizable_url(raw)) is None
+
+
+def test_origin_authorizable_url_leaves_ordinary_urls_untouched() -> None:
+ for url in ("https://host.example/path", "https://host.example/p?x=1&y=2", "https://host.example/"):
+ assert _origin_authorizable_url(url) == url
+
+
+# --- real production authorizer + real re-fetch chain ------------------------------------------
+
+
+async def test_production_authorizer_saves_benign_query_backslash(local_server: Any) -> None:
+ base, handler = local_server
+ url = f"{base}/ELABSTelFileDownload.aspx?TelFile={_WINDOWS_QUERY}"
+ response = await fetch_file_bytes(
+ url,
+ authorize_request_hop=_authz(),
+ download_scope=_SCOPE,
+ approved_initial_url=url,
+ normalize_query_backslashes=True,
+ )
+ assert response.body == _SAMPLE_BODY and response.filename == "sample.bin"
+ # The run-scoped authorizer admitted the hop AND the server received the exact Windows path.
+ assert handler.received_raw_query, "the guarded re-fetch never reached the server"
+ assert urllib.parse.parse_qs(handler.received_raw_query[-1])["TelFile"] == [_WINDOWS_QUERY]
+
+
+async def test_production_authorizer_saves_already_encoded_query(local_server: Any) -> None:
+ base, handler = local_server
+ url = f"{base}/dl.aspx?TelFile={_ENCODED_QUERY}"
+ response = await fetch_file_bytes(
+ url,
+ authorize_request_hop=_authz(),
+ download_scope=_SCOPE,
+ approved_initial_url=url,
+ normalize_query_backslashes=True,
+ )
+ assert response.body == _SAMPLE_BODY
+ # Already-encoded %5C is not double-encoded; the server decodes it back to the backslash path.
+ assert urllib.parse.parse_qs(handler.received_raw_query[-1])["TelFile"] == [_WINDOWS_QUERY]
+
+
+async def test_default_off_rejected_at_precheck(local_server: Any) -> None:
+ base, handler = local_server
+ url = f"{base}/dl.aspx?TelFile={_WINDOWS_QUERY}"
+ # Default (opt-in off): the precheck refuses before any network request or authorizer call.
+ with pytest.raises(HttpException, match="browser-canonicalizable"):
+ await fetch_file_bytes(url, authorize_request_hop=_authz(), download_scope=_SCOPE, approved_initial_url=url)
+ assert handler.received_raw_query == []
+
+
+async def test_wrong_first_target_binding_rejected(local_server: Any) -> None:
+ base, handler = local_server
+ port = int(urllib.parse.urlsplit(base).port)
+ url = f"{base}/dl.aspx?TelFile={_WINDOWS_QUERY}"
+ # approved_initial_url has a DIFFERENT origin (port); the run-scoped first-hop binding must refuse,
+ # proving the opt-in did not weaken the target/initial origin binding.
+ other_initial = f"http://127.0.0.1:{port + 1}/dl.aspx?TelFile={_WINDOWS_QUERY}"
+ with pytest.raises(PermissionError):
+ await fetch_file_bytes(
+ url,
+ authorize_request_hop=_authz(),
+ download_scope=_SCOPE,
+ approved_initial_url=other_initial,
+ normalize_query_backslashes=True,
+ )
+ assert handler.received_raw_query == []
+
+
+async def test_disallowed_redirect_fails_closed(local_server: Any) -> None:
+ base, _ = local_server
+ url = f"{base}/redirect-denied?TelFile={_WINDOWS_QUERY}"
+ # First hop is admitted (same origin) and reaches the server, which 302s to a link-local metadata
+ # address; the redirect hop is still validated and pinned, so the denied origin fails closed.
+ with pytest.raises(Exception) as excinfo:
+ await fetch_file_bytes(
+ url,
+ authorize_request_hop=_authz(),
+ download_scope=_SCOPE,
+ approved_initial_url=url,
+ normalize_query_backslashes=True,
+ )
+ assert type(excinfo.value).__name__ in {"BlockedHost", "SkyvernHTTPException", "HttpException"}
+
+
+async def test_ssrf_target_refused_with_opt_in() -> None:
+ # A link-local metadata address is refused by the host guard before the authorizer, even opted in.
+ with pytest.raises(Exception) as excinfo:
+ await fetch_file_bytes(
+ "http://169.254.169.254/latest/meta-data?p=\\x\\y",
+ authorize_request_hop=_authz(),
+ download_scope=_SCOPE,
+ approved_initial_url="http://169.254.169.254/latest/meta-data?p=\\x\\y",
+ normalize_query_backslashes=True,
+ )
+ assert type(excinfo.value).__name__ in {"BlockedHost", "SkyvernHTTPException", "HttpException"}
+
+
+async def test_ordinary_no_backslash_default_saves(local_server: Any) -> None:
+ base, _ = local_server
+ url = f"{base}/dl.aspx?x=1"
+ response = await fetch_file_bytes(
+ url, authorize_request_hop=_authz(), download_scope=_SCOPE, approved_initial_url=url
+ )
+ assert response.body == _SAMPLE_BODY