🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): ignore a leading byte-order mark in robots.txt and CSV feeds (#453)
Suliman Abdulrazzaq6ee33221a256
diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.pyindex 1f2194e..1afdd93 100644--- a/scrapling/spiders/robotstxt.py+++ b/scrapling/spiders/robotstxt.py@@ -38,7 +38,8 @@ class RobotsTxtManager:content = "User-agent: *\nDisallow: /"try:- parser = Protego.parse(content)+ # Left in place, a BOM hides the first line (usually `User-agent: *`) from protego+ parser = Protego.parse(content.removeprefix("\N{BYTE ORDER MARK}"))except Exception as e:log.warning(f"Failed to parse robots.txt for {domain}: {e}")parser = Protego.parse("")diff --git a/scrapling/spiders/templates/feed.py b/scrapling/spiders/templates/feed.pyindex ee0e939..ee009d8 100644--- a/scrapling/spiders/templates/feed.py+++ b/scrapling/spiders/templates/feed.py@@ -126,7 +126,8 @@ class CSVFeedSpider(Spider):self.logger.warning(f"Failed to decompress feed: {e}")return- text = body.decode(response.encoding or "utf-8", errors="replace")+ # Spreadsheet exports often start with a BOM, which would otherwise end up in the first column name+ text = body.decode(response.encoding or "utf-8", errors="replace").removeprefix("\N{BYTE ORDER MARK}")reader = DictReader(StringIO(text), fieldnames=self.headers, delimiter=self.delimiter, quotechar=self.quotechar)for row in reader:async for result in self.parse_row(response, dict(row)):diff --git a/scrapling/spiders/templates/sitemap.py b/scrapling/spiders/templates/sitemap.pyindex dcd947f..a16db62 100644--- a/scrapling/spiders/templates/sitemap.py+++ b/scrapling/spiders/templates/sitemap.py@@ -79,7 +79,8 @@ class SitemapSpider(Spider):"""Extract `Sitemap` directives from a robots.txt body via protego."""try:text = response.body.decode(response.encoding, errors="replace")- parser = Protego.parse(text)+ # Left in place, a BOM hides the first directive from protego+ parser = Protego.parse(text.removeprefix("\N{BYTE ORDER MARK}"))except Exception as e:self.logger.warning(f"Failed to parse robots.txt: {e}")return []diff --git a/tests/spiders/test_feed.py b/tests/spiders/test_feed.pyindex a2dae37..c4be1b4 100644--- a/tests/spiders/test_feed.py+++ b/tests/spiders/test_feed.py@@ -231,6 +231,11 @@ class TestCSVFeedSpider:rows = await _collect(_PriceSpider().parse(_make_response(gzip.compress(CSV_BODY))))assert len(rows) == 2+ @pytest.mark.asyncio+ async def test_leading_byte_order_mark_stays_out_of_the_first_column(self):+ rows = await _collect(_PriceSpider().parse(_make_response(b"\xef\xbb\xbf" + CSV_BODY)))+ assert rows[0] == {"title": "First", "price": "10.5", "url": "https://example.com/products/1"}+@pytest.mark.asyncioasync def test_empty_body_yields_nothing(self):assert await _collect(_PriceSpider().parse(_make_response(b""))) == []diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.pyindex 94bb288..9e238bb 100644--- a/tests/spiders/test_robotstxt.py+++ b/tests/spiders/test_robotstxt.py@@ -442,6 +442,20 @@ class TestEncoding:assert await mgr.can_fetch("https://example.com/private/", "s1") is Falseassert await mgr.can_fetch("https://example.com/public/", "s1") is True+ @pytest.mark.asyncio+ async def test_leading_byte_order_mark_is_ignored(self):+ body = b"\xef\xbb\xbf" + ROBOTS_BASIC.encode("utf-8")++ async def fetch_fn(url: str, sid: str) -> MockResponse:+ return MockResponse(status=200, body=body, encoding="utf-8")++ mgr = RobotsTxtManager(fetch_fn)++ assert await mgr.can_fetch("https://example.com/admin/", "s1") is False+ assert await mgr.can_fetch("https://example.com/products", "s1") is True+ c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")+ assert c_delay == 2.0+# ---------------------------------------------------------------------------diff --git a/tests/spiders/test_sitemap.py b/tests/spiders/test_sitemap.pyindex 362dd31..725fff6 100644--- a/tests/spiders/test_sitemap.py+++ b/tests/spiders/test_sitemap.py@@ -214,6 +214,18 @@ class TestRobotsTxt:assert out[0].url == "https://example.com/sitemap.xml"assert out[0].callback == spider._parse_sitemap+ @pytest.mark.asyncio+ async def test_parse_sitemap_robots_with_leading_byte_order_mark(self):+ class S(SitemapSpider):+ name = "s"+ sitemap_urls = ["https://example.com/robots.txt"]++ spider = S()+ body = b"\xef\xbb\xbfSitemap: https://example.com/sitemap.xml\nUser-agent: *\nDisallow: /admin/\n"+ resp = _make_response(body, url="https://example.com/robots.txt")+ out = await _collect(spider._parse_sitemap(resp))+ assert [req.url for req in out] == ["https://example.com/sitemap.xml"]+@pytest.mark.asyncioasync def test_parse_sitemap_robots_with_no_directives_warns(self):# Spider's logger has propagate=False, so we attach our own handler to it.