Scrapy, a fast high-level web crawling & scraping framework for Python.
Move the referrer policy scenarios to module-level constants. (#8228)
Andrey Rakhmatullin8006f337cdca
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 5ca27d15c..2406495bd 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -1,7 +1,7 @@from __future__ import annotationsimport warnings-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, TypeAliasfrom urllib.parse import urlparseimport pytest@@ -66,567 +66,586 @@ class TestRefererMiddleware:assert out[0].headers.get("Referer") == referrer-class MixinDefault:- """- Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade+ScenarioTable: TypeAlias = list[tuple[str, str, bytes | None]]++# Referrer policy scenarios, as (origin, target, expected Referer) triples.++# Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade+# with some additional filtering of s3://+SCENARII_DEFAULT: ScenarioTable = [+ ("https://example.com/", "https://scrapy.org/", b"https://example.com/"),+ ("http://example.com/", "http://scrapy.org/", b"http://example.com/"),+ ("http://example.com/", "https://scrapy.org/", b"http://example.com/"),+ ("https://example.com/", "http://scrapy.org/", None),+ # no credentials leak+ (+ "http://user:[email protected]/",+ "https://scrapy.org/",+ b"http://example.com/",+ ),+ # no referrer leak for local schemes+ ("file:///home/path/to/somefile.html", "https://scrapy.org/", None),+ ("file:///home/path/to/somefile.html", "http://scrapy.org/", None),+ # no referrer leak for s3 origins+ ("s3://mybucket/path/to/data.csv", "https://scrapy.org/", None),+ ("s3://mybucket/path/to/data.csv", "http://scrapy.org/", None),+]++SCENARII_NO_REFERRER: ScenarioTable = [+ ("https://example.com/page.html", "https://example.com/", None),+ ("http://www.example.com/", "https://scrapy.org/", None),+ ("http://www.example.com/", "http://scrapy.org/", None),+ ("https://www.example.com/", "http://scrapy.org/", None),+ ("file:///home/path/to/somefile.html", "http://scrapy.org/", None),+]++SCENARII_NO_REFERRER_WHEN_DOWNGRADE: ScenarioTable = [+ # TLS to TLS: send non-empty referrer+ (+ "https://example.com/page.html",+ "https://not.example.com/",+ b"https://example.com/page.html",+ ),+ (+ "https://example.com/page.html",+ "https://scrapy.org/",+ b"https://example.com/page.html",+ ),+ (+ "https://example.com:443/page.html",+ "https://scrapy.org/",+ b"https://example.com/page.html",+ ),+ (+ "https://example.com:444/page.html",+ "https://scrapy.org/",+ b"https://example.com:444/page.html",+ ),+ (+ "ftps://example.com/urls.zip",+ "https://scrapy.org/",+ b"ftps://example.com/urls.zip",+ ),+ # TLS to non-TLS: do not send referrer+ ("https://example.com/page.html", "http://not.example.com/", None),+ ("https://example.com/page.html", "http://scrapy.org/", None),+ ("ftps://example.com/urls.zip", "http://scrapy.org/", None),+ # non-TLS to TLS or non-TLS: send referrer+ (+ "http://example.com/page.html",+ "https://not.example.com/",+ b"http://example.com/page.html",+ ),+ (+ "http://example.com/page.html",+ "https://scrapy.org/",+ b"http://example.com/page.html",+ ),+ (+ "http://example.com:8080/page.html",+ "https://scrapy.org/",+ b"http://example.com:8080/page.html",+ ),+ (+ "http://example.com:80/page.html",+ "http://not.example.com/",+ b"http://example.com/page.html",+ ),+ (+ "http://example.com/page.html",+ "http://scrapy.org/",+ b"http://example.com/page.html",+ ),+ (+ "http://example.com:443/page.html",+ "http://scrapy.org/",+ b"http://example.com:443/page.html",+ ),+ (+ "ftp://example.com/urls.zip",+ "http://scrapy.org/",+ b"ftp://example.com/urls.zip",+ ),+ (+ "ftp://example.com/urls.zip",+ "https://scrapy.org/",+ b"ftp://example.com/urls.zip",+ ),+ # test for user/password stripping+ (+ "http://user:[email protected]/page.html",+ "https://not.example.com/",+ b"http://example.com/page.html",+ ),+]++SCENARII_SAME_ORIGIN: ScenarioTable = [+ # Same origin (protocol, host, port): send referrer+ (+ "https://example.com/page.html",+ "https://example.com/not-page.html",+ b"https://example.com/page.html",+ ),+ (+ "http://example.com/page.html",+ "http://example.com/not-page.html",+ b"http://example.com/page.html",+ ),+ (+ "https://example.com:443/page.html",+ "https://example.com/not-page.html",+ b"https://example.com/page.html",+ ),+ (+ "http://example.com:80/page.html",+ "http://example.com/not-page.html",+ b"http://example.com/page.html",+ ),+ (+ "http://example.com/page.html",+ "http://example.com:80/not-page.html",+ b"http://example.com/page.html",+ ),+ (+ "http://example.com:8888/page.html",+ "http://example.com:8888/not-page.html",+ b"http://example.com:8888/page.html",+ ),+ # Different host: do NOT send referrer+ (+ "https://example.com/page.html",+ "https://not.example.com/otherpage.html",+ None,+ ),+ ("http://example.com/page.html", "http://not.example.com/otherpage.html", None),+ ("http://example.com/page.html", "http://www.example.com/otherpage.html", None),+ # Different port: do NOT send referrer+ (+ "https://example.com:444/page.html",+ "https://example.com/not-page.html",+ None,+ ),+ ("http://example.com:81/page.html", "http://example.com/not-page.html", None),+ ("http://example.com/page.html", "http://example.com:81/not-page.html", None),+ # Different protocols: do NOT send referrer+ ("https://example.com/page.html", "http://example.com/not-page.html", None),+ ("https://example.com/page.html", "http://not.example.com/", None),+ ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),+ ("ftp://example.com/urls.zip", "http://example.com/not-page.html", None),+ ("ftps://example.com/urls.zip", "http://example.com/not-page.html", None),+ # test for user/password stripping+ (+ "https://user:[email protected]/page.html",+ "http://example.com/not-page.html",+ None,+ ),+ (+ "https://user:[email protected]/page.html",+ "https://example.com/not-page.html",+ b"https://example.com/page.html",+ ),+]++SCENARII_ORIGIN: ScenarioTable = [+ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)+ (+ "https://example.com/page.html",+ "https://example.com/not-page.html",+ b"https://example.com/",+ ),+ (+ "https://example.com/page.html",+ "https://scrapy.org",+ b"https://example.com/",+ ),+ ("https://example.com/page.html", "http://scrapy.org", b"https://example.com/"),+ ("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"),+ # test for user/password stripping+ (+ "https://user:[email protected]/page.html",+ "http://scrapy.org",+ b"https://example.com/",+ ),+]++SCENARII_STRICT_ORIGIN: ScenarioTable = [+ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades+ (+ "https://example.com/page.html",+ "https://example.com/not-page.html",+ b"https://example.com/",+ ),+ (+ "https://example.com/page.html",+ "https://scrapy.org",+ b"https://example.com/",+ ),+ ("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"),+ # downgrade: send nothing+ ("https://example.com/page.html", "http://scrapy.org", None),+ # upgrade: send origin+ ("http://example.com/page.html", "https://scrapy.org", b"http://example.com/"),+ # test for user/password stripping+ (+ "https://user:[email protected]/page.html",+ "https://scrapy.org",+ b"https://example.com/",+ ),+ ("https://user:[email protected]/page.html", "http://scrapy.org", None),+]++SCENARII_ORIGIN_WHEN_CROSS_ORIGIN: ScenarioTable = [+ # Same origin (protocol, host, port): send referrer+ (+ "https://example.com/page.html",+ "https://example.com/not-page.html",+ b"https://example.com/page.html",+ ),+ (+ "http://example.com/page.html",+ "http://example.com/not-page.html",+ b"http://example.com/page.html",+ ),+ (+ "https://example.com:443/page.html",+ "https://example.com/not-page.html",+ b"https://example.com/page.html",+ ),+ (+ "http://example.com:80/page.html",+ "http://example.com/not-page.html",+ b"http://example.com/page.html",+ ),+ (+ "http://example.com/page.html",+ "http://example.com:80/not-page.html",+ b"http://example.com/page.html",+ ),+ (+ "http://example.com:8888/page.html",+ "http://example.com:8888/not-page.html",+ b"http://example.com:8888/page.html",+ ),+ # Different host: send origin as referrer+ (+ "https://example2.com/page.html",+ "https://scrapy.org/otherpage.html",+ b"https://example2.com/",+ ),+ (+ "https://example2.com/page.html",+ "https://not.example2.com/otherpage.html",+ b"https://example2.com/",+ ),+ (+ "http://example2.com/page.html",+ "http://not.example2.com/otherpage.html",+ b"http://example2.com/",+ ),+ # exact match required+ (+ "http://example2.com/page.html",+ "http://www.example2.com/otherpage.html",+ b"http://example2.com/",+ ),+ # Different port: send origin as referrer+ (+ "https://example3.com:444/page.html",+ "https://example3.com/not-page.html",+ b"https://example3.com:444/",+ ),+ (+ "http://example3.com:81/page.html",+ "http://example3.com/not-page.html",+ b"http://example3.com:81/",+ ),+ # Different protocols: send origin as referrer+ (+ "https://example4.com/page.html",+ "http://example4.com/not-page.html",+ b"https://example4.com/",+ ),+ (+ "https://example4.com/page.html",+ "http://not.example4.com/",+ b"https://example4.com/",+ ),+ (+ "ftps://example4.com/urls.zip",+ "https://example4.com/not-page.html",+ b"ftps://example4.com/",+ ),+ (+ "ftp://example4.com/urls.zip",+ "http://example4.com/not-page.html",+ b"ftp://example4.com/",+ ),+ (+ "ftps://example4.com/urls.zip",+ "http://example4.com/not-page.html",+ b"ftps://example4.com/",+ ),+ # test for user/password stripping+ (+ "https://user:[email protected]/page.html",+ "https://example5.com/not-page.html",+ b"https://example5.com/page.html",+ ),+ # TLS to non-TLS downgrade: send origin+ (+ "https://user:[email protected]/page.html",+ "http://example5.com/not-page.html",+ b"https://example5.com/",+ ),+]++SCENARII_STRICT_ORIGIN_WHEN_CROSS_ORIGIN: ScenarioTable = [+ # Same origin (protocol, host, port): send referrer+ (+ "https://example.com/page.html",+ "https://example.com/not-page.html",+ b"https://example.com/page.html",+ ),+ (+ "http://example.com/page.html",+ "http://example.com/not-page.html",+ b"http://example.com/page.html",+ ),+ (+ "https://example.com:443/page.html",+ "https://example.com/not-page.html",+ b"https://example.com/page.html",+ ),+ (+ "http://example.com:80/page.html",+ "http://example.com/not-page.html",+ b"http://example.com/page.html",+ ),+ (+ "http://example.com/page.html",+ "http://example.com:80/not-page.html",+ b"http://example.com/page.html",+ ),+ (+ "http://example.com:8888/page.html",+ "http://example.com:8888/not-page.html",+ b"http://example.com:8888/page.html",+ ),+ # Different host: send origin as referrer+ (+ "https://example2.com/page.html",+ "https://scrapy.org/otherpage.html",+ b"https://example2.com/",+ ),+ (+ "https://example2.com/page.html",+ "https://not.example2.com/otherpage.html",+ b"https://example2.com/",+ ),+ (+ "http://example2.com/page.html",+ "http://not.example2.com/otherpage.html",+ b"http://example2.com/",+ ),+ # exact match required+ (+ "http://example2.com/page.html",+ "http://www.example2.com/otherpage.html",+ b"http://example2.com/",+ ),+ # Different port: send origin as referrer+ (+ "https://example3.com:444/page.html",+ "https://example3.com/not-page.html",+ b"https://example3.com:444/",+ ),+ (+ "http://example3.com:81/page.html",+ "http://example3.com/not-page.html",+ b"http://example3.com:81/",+ ),+ # downgrade+ ("https://example4.com/page.html", "http://example4.com/not-page.html", None),+ ("https://example4.com/page.html", "http://not.example4.com/", None),+ # non-TLS to non-TLS+ (+ "ftp://example4.com/urls.zip",+ "http://example4.com/not-page.html",+ b"ftp://example4.com/",+ ),+ # upgrade+ (+ "http://example4.com/page.html",+ "https://example4.com/not-page.html",+ b"http://example4.com/",+ ),+ (+ "http://example4.com/page.html",+ "https://not.example4.com/",+ b"http://example4.com/",+ ),+ # Different protocols: send origin as referrer+ (+ "ftps://example4.com/urls.zip",+ "https://example4.com/not-page.html",+ b"ftps://example4.com/",+ ),+ (+ "ftp://example4.com/urls.zip",+ "http://example4.com/not-page.html",+ b"ftp://example4.com/",+ ),+ # test for user/password stripping+ (+ "https://user:[email protected]/page.html",+ "https://example5.com/not-page.html",+ b"https://example5.com/page.html",+ ),+ # TLS to non-TLS downgrade: send nothing+ (+ "https://user:[email protected]/page.html",+ "http://example5.com/not-page.html",+ None,+ ),+]++SCENARII_UNSAFE_URL: ScenarioTable = [+ # TLS to TLS: send referrer+ (+ "https://example.com/sekrit.html",+ "http://not.example.com/",+ b"https://example.com/sekrit.html",+ ),+ (+ "https://example1.com/page.html",+ "https://not.example1.com/",+ b"https://example1.com/page.html",+ ),+ (+ "https://example1.com/page.html",+ "https://scrapy.org/",+ b"https://example1.com/page.html",+ ),+ (+ "https://example1.com:443/page.html",+ "https://scrapy.org/",+ b"https://example1.com/page.html",+ ),+ (+ "https://example1.com:444/page.html",+ "https://scrapy.org/",+ b"https://example1.com:444/page.html",+ ),+ (+ "ftps://example1.com/urls.zip",+ "https://scrapy.org/",+ b"ftps://example1.com/urls.zip",+ ),+ # TLS to non-TLS: send referrer (yes, it's unsafe)+ (+ "https://example2.com/page.html",+ "http://not.example2.com/",+ b"https://example2.com/page.html",+ ),+ (+ "https://example2.com/page.html",+ "http://scrapy.org/",+ b"https://example2.com/page.html",+ ),+ (+ "ftps://example2.com/urls.zip",+ "http://scrapy.org/",+ b"ftps://example2.com/urls.zip",+ ),+ # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)+ (+ "http://example3.com/page.html",+ "https://not.example3.com/",+ b"http://example3.com/page.html",+ ),+ (+ "http://example3.com/page.html",+ "https://scrapy.org/",+ b"http://example3.com/page.html",+ ),+ (+ "http://example3.com:8080/page.html",+ "https://scrapy.org/",+ b"http://example3.com:8080/page.html",+ ),+ (+ "http://example3.com:80/page.html",+ "http://not.example3.com/",+ b"http://example3.com/page.html",+ ),+ (+ "http://example3.com/page.html",+ "http://scrapy.org/",+ b"http://example3.com/page.html",+ ),+ (+ "http://example3.com:443/page.html",+ "http://scrapy.org/",+ b"http://example3.com:443/page.html",+ ),+ (+ "ftp://example3.com/urls.zip",+ "http://scrapy.org/",+ b"ftp://example3.com/urls.zip",+ ),+ (+ "ftp://example3.com/urls.zip",+ "https://scrapy.org/",+ b"ftp://example3.com/urls.zip",+ ),+ # test for user/password stripping+ (+ "http://user:[email protected]/page.html",+ "https://not.example4.com/",+ b"http://example4.com/page.html",+ ),+ (+ "https://user:[email protected]/page.html",+ "http://scrapy.org/",+ b"https://example4.com/page.html",+ ),+]- with some additional filtering of s3://- """- scenarii: list[tuple[str, str, bytes | None]] = [- ("https://example.com/", "https://scrapy.org/", b"https://example.com/"),- ("http://example.com/", "http://scrapy.org/", b"http://example.com/"),- ("http://example.com/", "https://scrapy.org/", b"http://example.com/"),- ("https://example.com/", "http://scrapy.org/", None),- # no credentials leak- (- "http://user:[email protected]/",- "https://scrapy.org/",- b"http://example.com/",- ),- # no referrer leak for local schemes- ("file:///home/path/to/somefile.html", "https://scrapy.org/", None),- ("file:///home/path/to/somefile.html", "http://scrapy.org/", None),- # no referrer leak for s3 origins- ("s3://mybucket/path/to/data.csv", "https://scrapy.org/", None),- ("s3://mybucket/path/to/data.csv", "http://scrapy.org/", None),- ]+class MixinDefault:+ scenarii = SCENARII_DEFAULTclass MixinNoReferrer:- scenarii: list[tuple[str, str, bytes | None]] = [- ("https://example.com/page.html", "https://example.com/", None),- ("http://www.example.com/", "https://scrapy.org/", None),- ("http://www.example.com/", "http://scrapy.org/", None),- ("https://www.example.com/", "http://scrapy.org/", None),- ("file:///home/path/to/somefile.html", "http://scrapy.org/", None),- ]+ scenarii = SCENARII_NO_REFERRERclass MixinNoReferrerWhenDowngrade:- scenarii: list[tuple[str, str, bytes | None]] = [- # TLS to TLS: send non-empty referrer- (- "https://example.com/page.html",- "https://not.example.com/",- b"https://example.com/page.html",- ),- (- "https://example.com/page.html",- "https://scrapy.org/",- b"https://example.com/page.html",- ),- (- "https://example.com:443/page.html",- "https://scrapy.org/",- b"https://example.com/page.html",- ),- (- "https://example.com:444/page.html",- "https://scrapy.org/",- b"https://example.com:444/page.html",- ),- (- "ftps://example.com/urls.zip",- "https://scrapy.org/",- b"ftps://example.com/urls.zip",- ),- # TLS to non-TLS: do not send referrer- ("https://example.com/page.html", "http://not.example.com/", None),- ("https://example.com/page.html", "http://scrapy.org/", None),- ("ftps://example.com/urls.zip", "http://scrapy.org/", None),- # non-TLS to TLS or non-TLS: send referrer- (- "http://example.com/page.html",- "https://not.example.com/",- b"http://example.com/page.html",- ),- (- "http://example.com/page.html",- "https://scrapy.org/",- b"http://example.com/page.html",- ),- (- "http://example.com:8080/page.html",- "https://scrapy.org/",- b"http://example.com:8080/page.html",- ),- (- "http://example.com:80/page.html",- "http://not.example.com/",- b"http://example.com/page.html",- ),- (- "http://example.com/page.html",- "http://scrapy.org/",- b"http://example.com/page.html",- ),- (- "http://example.com:443/page.html",- "http://scrapy.org/",- b"http://example.com:443/page.html",- ),- (- "ftp://example.com/urls.zip",- "http://scrapy.org/",- b"ftp://example.com/urls.zip",- ),- (- "ftp://example.com/urls.zip",- "https://scrapy.org/",- b"ftp://example.com/urls.zip",- ),- # test for user/password stripping- (- "http://user:[email protected]/page.html",- "https://not.example.com/",- b"http://example.com/page.html",- ),- ]+ scenarii = SCENARII_NO_REFERRER_WHEN_DOWNGRADEclass MixinSameOrigin:- scenarii: list[tuple[str, str, bytes | None]] = [- # Same origin (protocol, host, port): send referrer- (- "https://example.com/page.html",- "https://example.com/not-page.html",- b"https://example.com/page.html",- ),- (- "http://example.com/page.html",- "http://example.com/not-page.html",- b"http://example.com/page.html",- ),- (- "https://example.com:443/page.html",- "https://example.com/not-page.html",- b"https://example.com/page.html",- ),- (- "http://example.com:80/page.html",- "http://example.com/not-page.html",- b"http://example.com/page.html",- ),- (- "http://example.com/page.html",- "http://example.com:80/not-page.html",- b"http://example.com/page.html",- ),- (- "http://example.com:8888/page.html",- "http://example.com:8888/not-page.html",- b"http://example.com:8888/page.html",- ),- # Different host: do NOT send referrer- (- "https://example.com/page.html",- "https://not.example.com/otherpage.html",- None,- ),- ("http://example.com/page.html", "http://not.example.com/otherpage.html", None),- ("http://example.com/page.html", "http://www.example.com/otherpage.html", None),- # Different port: do NOT send referrer- (- "https://example.com:444/page.html",- "https://example.com/not-page.html",- None,- ),- ("http://example.com:81/page.html", "http://example.com/not-page.html", None),- ("http://example.com/page.html", "http://example.com:81/not-page.html", None),- # Different protocols: do NOT send referrer- ("https://example.com/page.html", "http://example.com/not-page.html", None),- ("https://example.com/page.html", "http://not.example.com/", None),- ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),- ("ftp://example.com/urls.zip", "http://example.com/not-page.html", None),- ("ftps://example.com/urls.zip", "http://example.com/not-page.html", None),- # test for user/password stripping- (- "https://user:[email protected]/page.html",- "http://example.com/not-page.html",- None,- ),- (- "https://user:[email protected]/page.html",- "https://example.com/not-page.html",- b"https://example.com/page.html",- ),- ]+ scenarii = SCENARII_SAME_ORIGINclass MixinOrigin:- scenarii: list[tuple[str, str, bytes | None]] = [- # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)- (- "https://example.com/page.html",- "https://example.com/not-page.html",- b"https://example.com/",- ),- (- "https://example.com/page.html",- "https://scrapy.org",- b"https://example.com/",- ),- ("https://example.com/page.html", "http://scrapy.org", b"https://example.com/"),- ("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"),- # test for user/password stripping- (- "https://user:[email protected]/page.html",- "http://scrapy.org",- b"https://example.com/",- ),- ]+ scenarii = SCENARII_ORIGINclass MixinStrictOrigin:- scenarii: list[tuple[str, str, bytes | None]] = [- # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades- (- "https://example.com/page.html",- "https://example.com/not-page.html",- b"https://example.com/",- ),- (- "https://example.com/page.html",- "https://scrapy.org",- b"https://example.com/",- ),- ("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"),- # downgrade: send nothing- ("https://example.com/page.html", "http://scrapy.org", None),- # upgrade: send origin- ("http://example.com/page.html", "https://scrapy.org", b"http://example.com/"),- # test for user/password stripping- (- "https://user:[email protected]/page.html",- "https://scrapy.org",- b"https://example.com/",- ),- ("https://user:[email protected]/page.html", "http://scrapy.org", None),- ]+ scenarii = SCENARII_STRICT_ORIGINclass MixinOriginWhenCrossOrigin:- scenarii: list[tuple[str, str, bytes | None]] = [- # Same origin (protocol, host, port): send referrer- (- "https://example.com/page.html",- "https://example.com/not-page.html",- b"https://example.com/page.html",- ),- (- "http://example.com/page.html",- "http://example.com/not-page.html",- b"http://example.com/page.html",- ),- (- "https://example.com:443/page.html",- "https://example.com/not-page.html",- b"https://example.com/page.html",- ),- (- "http://example.com:80/page.html",- "http://example.com/not-page.html",- b"http://example.com/page.html",- ),- (- "http://example.com/page.html",- "http://example.com:80/not-page.html",- b"http://example.com/page.html",- ),- (- "http://example.com:8888/page.html",- "http://example.com:8888/not-page.html",- b"http://example.com:8888/page.html",- ),- # Different host: send origin as referrer- (- "https://example2.com/page.html",- "https://scrapy.org/otherpage.html",- b"https://example2.com/",- ),- (- "https://example2.com/page.html",- "https://not.example2.com/otherpage.html",- b"https://example2.com/",- ),- (- "http://example2.com/page.html",- "http://not.example2.com/otherpage.html",- b"http://example2.com/",- ),- # exact match required- (- "http://example2.com/page.html",- "http://www.example2.com/otherpage.html",- b"http://example2.com/",- ),- # Different port: send origin as referrer- (- "https://example3.com:444/page.html",- "https://example3.com/not-page.html",- b"https://example3.com:444/",- ),- (- "http://example3.com:81/page.html",- "http://example3.com/not-page.html",- b"http://example3.com:81/",- ),- # Different protocols: send origin as referrer- (- "https://example4.com/page.html",- "http://example4.com/not-page.html",- b"https://example4.com/",- ),- (- "https://example4.com/page.html",- "http://not.example4.com/",- b"https://example4.com/",- ),- (- "ftps://example4.com/urls.zip",- "https://example4.com/not-page.html",- b"ftps://example4.com/",- ),- (- "ftp://example4.com/urls.zip",- "http://example4.com/not-page.html",- b"ftp://example4.com/",- ),- (- "ftps://example4.com/urls.zip",- "http://example4.com/not-page.html",- b"ftps://example4.com/",- ),- # test for user/password stripping- (- "https://user:[email protected]/page.html",- "https://example5.com/not-page.html",- b"https://example5.com/page.html",- ),- # TLS to non-TLS downgrade: send origin- (- "https://user:[email protected]/page.html",- "http://example5.com/not-page.html",- b"https://example5.com/",- ),- ]+ scenarii = SCENARII_ORIGIN_WHEN_CROSS_ORIGINclass MixinStrictOriginWhenCrossOrigin:- scenarii: list[tuple[str, str, bytes | None]] = [- # Same origin (protocol, host, port): send referrer- (- "https://example.com/page.html",- "https://example.com/not-page.html",- b"https://example.com/page.html",- ),- (- "http://example.com/page.html",- "http://example.com/not-page.html",- b"http://example.com/page.html",- ),- (- "https://example.com:443/page.html",- "https://example.com/not-page.html",- b"https://example.com/page.html",- ),- (- "http://example.com:80/page.html",- "http://example.com/not-page.html",- b"http://example.com/page.html",- ),- (- "http://example.com/page.html",- "http://example.com:80/not-page.html",- b"http://example.com/page.html",- ),- (- "http://example.com:8888/page.html",- "http://example.com:8888/not-page.html",- b"http://example.com:8888/page.html",- ),- # Different host: send origin as referrer- (- "https://example2.com/page.html",- "https://scrapy.org/otherpage.html",- b"https://example2.com/",- ),- (- "https://example2.com/page.html",- "https://not.example2.com/otherpage.html",- b"https://example2.com/",- ),- (- "http://example2.com/page.html",- "http://not.example2.com/otherpage.html",- b"http://example2.com/",- ),- # exact match required- (- "http://example2.com/page.html",- "http://www.example2.com/otherpage.html",- b"http://example2.com/",- ),- # Different port: send origin as referrer- (- "https://example3.com:444/page.html",- "https://example3.com/not-page.html",- b"https://example3.com:444/",- ),- (- "http://example3.com:81/page.html",- "http://example3.com/not-page.html",- b"http://example3.com:81/",- ),- # downgrade- ("https://example4.com/page.html", "http://example4.com/not-page.html", None),- ("https://example4.com/page.html", "http://not.example4.com/", None),- # non-TLS to non-TLS- (- "ftp://example4.com/urls.zip",- "http://example4.com/not-page.html",- b"ftp://example4.com/",- ),- # upgrade- (- "http://example4.com/page.html",- "https://example4.com/not-page.html",- b"http://example4.com/",- ),- (- "http://example4.com/page.html",- "https://not.example4.com/",- b"http://example4.com/",- ),- # Different protocols: send origin as referrer- (- "ftps://example4.com/urls.zip",- "https://example4.com/not-page.html",- b"ftps://example4.com/",- ),- (- "ftp://example4.com/urls.zip",- "http://example4.com/not-page.html",- b"ftp://example4.com/",- ),- # test for user/password stripping- (- "https://user:[email protected]/page.html",- "https://example5.com/not-page.html",- b"https://example5.com/page.html",- ),- # TLS to non-TLS downgrade: send nothing- (- "https://user:[email protected]/page.html",- "http://example5.com/not-page.html",- None,- ),- ]+ scenarii = SCENARII_STRICT_ORIGIN_WHEN_CROSS_ORIGINclass MixinUnsafeUrl:- scenarii: list[tuple[str, str, bytes | None]] = [- # TLS to TLS: send referrer- (- "https://example.com/sekrit.html",- "http://not.example.com/",- b"https://example.com/sekrit.html",- ),- (- "https://example1.com/page.html",- "https://not.example1.com/",- b"https://example1.com/page.html",- ),- (- "https://example1.com/page.html",- "https://scrapy.org/",- b"https://example1.com/page.html",- ),- (- "https://example1.com:443/page.html",- "https://scrapy.org/",- b"https://example1.com/page.html",- ),- (- "https://example1.com:444/page.html",- "https://scrapy.org/",- b"https://example1.com:444/page.html",- ),- (- "ftps://example1.com/urls.zip",- "https://scrapy.org/",- b"ftps://example1.com/urls.zip",- ),- # TLS to non-TLS: send referrer (yes, it's unsafe)- (- "https://example2.com/page.html",- "http://not.example2.com/",- b"https://example2.com/page.html",- ),- (- "https://example2.com/page.html",- "http://scrapy.org/",- b"https://example2.com/page.html",- ),- (- "ftps://example2.com/urls.zip",- "http://scrapy.org/",- b"ftps://example2.com/urls.zip",- ),- # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)- (- "http://example3.com/page.html",- "https://not.example3.com/",- b"http://example3.com/page.html",- ),- (- "http://example3.com/page.html",- "https://scrapy.org/",- b"http://example3.com/page.html",- ),- (- "http://example3.com:8080/page.html",- "https://scrapy.org/",- b"http://example3.com:8080/page.html",- ),- (- "http://example3.com:80/page.html",- "http://not.example3.com/",- b"http://example3.com/page.html",- ),- (- "http://example3.com/page.html",- "http://scrapy.org/",- b"http://example3.com/page.html",- ),- (- "http://example3.com:443/page.html",- "http://scrapy.org/",- b"http://example3.com:443/page.html",- ),- (- "ftp://example3.com/urls.zip",- "http://scrapy.org/",- b"ftp://example3.com/urls.zip",- ),- (- "ftp://example3.com/urls.zip",- "https://scrapy.org/",- b"ftp://example3.com/urls.zip",- ),- # test for user/password stripping- (- "http://user:[email protected]/page.html",- "https://not.example4.com/",- b"http://example4.com/page.html",- ),- (- "https://user:[email protected]/page.html",- "http://scrapy.org/",- b"https://example4.com/page.html",- ),- ]+ scenarii = SCENARII_UNSAFE_URLclass TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):