Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): ignore a leading byte-order mark in robots.txt and CSV feeds (#453)

Suliman Abdulrazzaq6ee33221a256
diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.py
index 1f2194e..1afdd93 100644
--- a/scrapling/spiders/robotstxt.py
+++ b/scrapling/spiders/robotstxt.py
@@ -38,7 +38,8 @@ class RobotsTxtManager:
content = "User-agent: *\nDisallow: /"
try:
- parser = Protego.parse(content)
+ # Left in place, a BOM hides the first line (usually `User-agent: *`) from protego
+ parser = Protego.parse(content.removeprefix("\N{BYTE ORDER MARK}"))
except Exception as e:
log.warning(f"Failed to parse robots.txt for {domain}: {e}")
parser = Protego.parse("")
diff --git a/scrapling/spiders/templates/feed.py b/scrapling/spiders/templates/feed.py
index ee0e939..ee009d8 100644
--- a/scrapling/spiders/templates/feed.py
+++ b/scrapling/spiders/templates/feed.py
@@ -126,7 +126,8 @@ class CSVFeedSpider(Spider):
self.logger.warning(f"Failed to decompress feed: {e}")
return
- text = body.decode(response.encoding or "utf-8", errors="replace")
+ # Spreadsheet exports often start with a BOM, which would otherwise end up in the first column name
+ text = body.decode(response.encoding or "utf-8", errors="replace").removeprefix("\N{BYTE ORDER MARK}")
reader = DictReader(StringIO(text), fieldnames=self.headers, delimiter=self.delimiter, quotechar=self.quotechar)
for row in reader:
async for result in self.parse_row(response, dict(row)):
diff --git a/scrapling/spiders/templates/sitemap.py b/scrapling/spiders/templates/sitemap.py
index dcd947f..a16db62 100644
--- a/scrapling/spiders/templates/sitemap.py
+++ b/scrapling/spiders/templates/sitemap.py
@@ -79,7 +79,8 @@ class SitemapSpider(Spider):
"""Extract `Sitemap` directives from a robots.txt body via protego."""
try:
text = response.body.decode(response.encoding, errors="replace")
- parser = Protego.parse(text)
+ # Left in place, a BOM hides the first directive from protego
+ parser = Protego.parse(text.removeprefix("\N{BYTE ORDER MARK}"))
except Exception as e:
self.logger.warning(f"Failed to parse robots.txt: {e}")
return []
diff --git a/tests/spiders/test_feed.py b/tests/spiders/test_feed.py
index a2dae37..c4be1b4 100644
--- a/tests/spiders/test_feed.py
+++ b/tests/spiders/test_feed.py
@@ -231,6 +231,11 @@ class TestCSVFeedSpider:
rows = await _collect(_PriceSpider().parse(_make_response(gzip.compress(CSV_BODY))))
assert len(rows) == 2
+ @pytest.mark.asyncio
+ async def test_leading_byte_order_mark_stays_out_of_the_first_column(self):
+ rows = await _collect(_PriceSpider().parse(_make_response(b"\xef\xbb\xbf" + CSV_BODY)))
+ assert rows[0] == {"title": "First", "price": "10.5", "url": "https://example.com/products/1"}
+
@pytest.mark.asyncio
async def test_empty_body_yields_nothing(self):
assert await _collect(_PriceSpider().parse(_make_response(b""))) == []
diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.py
index 94bb288..9e238bb 100644
--- a/tests/spiders/test_robotstxt.py
+++ b/tests/spiders/test_robotstxt.py
@@ -442,6 +442,20 @@ class TestEncoding:
assert await mgr.can_fetch("https://example.com/private/", "s1") is False
assert await mgr.can_fetch("https://example.com/public/", "s1") is True
+ @pytest.mark.asyncio
+ async def test_leading_byte_order_mark_is_ignored(self):
+ body = b"\xef\xbb\xbf" + ROBOTS_BASIC.encode("utf-8")
+
+ async def fetch_fn(url: str, sid: str) -> MockResponse:
+ return MockResponse(status=200, body=body, encoding="utf-8")
+
+ mgr = RobotsTxtManager(fetch_fn)
+
+ assert await mgr.can_fetch("https://example.com/admin/", "s1") is False
+ assert await mgr.can_fetch("https://example.com/products", "s1") is True
+ c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")
+ assert c_delay == 2.0
+
# ---------------------------------------------------------------------------
diff --git a/tests/spiders/test_sitemap.py b/tests/spiders/test_sitemap.py
index 362dd31..725fff6 100644
--- a/tests/spiders/test_sitemap.py
+++ b/tests/spiders/test_sitemap.py
@@ -214,6 +214,18 @@ class TestRobotsTxt:
assert out[0].url == "https://example.com/sitemap.xml"
assert out[0].callback == spider._parse_sitemap
+ @pytest.mark.asyncio
+ async def test_parse_sitemap_robots_with_leading_byte_order_mark(self):
+ class S(SitemapSpider):
+ name = "s"
+ sitemap_urls = ["https://example.com/robots.txt"]
+
+ spider = S()
+ body = b"\xef\xbb\xbfSitemap: https://example.com/sitemap.xml\nUser-agent: *\nDisallow: /admin/\n"
+ resp = _make_response(body, url="https://example.com/robots.txt")
+ out = await _collect(spider._parse_sitemap(resp))
+ assert [req.url for req in out] == ["https://example.com/sitemap.xml"]
+
@pytest.mark.asyncio
async def test_parse_sitemap_robots_with_no_directives_warns(self):
# Spider's logger has propagate=False, so we attach our own handler to it.