Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(web-source): scope crawl seeds to their parent directory

XZH482d62a0f65c
diff --git a/deeptutor/services/web_source/crawler.py b/deeptutor/services/web_source/crawler.py
index f00f9f4de..1e827ca56 100644
--- a/deeptutor/services/web_source/crawler.py
+++ b/deeptutor/services/web_source/crawler.py
@@ -381,7 +381,17 @@ async def crawl_docs_site(
result.errors.append(f"Disallowed host: {base_host}")
return result
- base_path_prefix = parsed.path or "/"
+ # Scope the crawl to the seed's directory so sibling pages count as
+ # internal: /intro/home seeds /intro/*, a root URL seeds the whole host.
+ # Pure URL-path rules (not pathlib) — Path("/intro/").parent is "/", which
+ # would silently widen a section seed to the entire host, and WindowsPath
+ # renders separators as "\", breaking the prefix entirely.
+ raw_path = parsed.path or "/"
+ if raw_path.endswith("/"):
+ # Directory index (or root): the directory itself is the scope.
+ base_path_prefix = raw_path
+ else:
+ base_path_prefix = raw_path.rsplit("/", 1)[0] or "/"
factory = client_factory or (
lambda: httpx.AsyncClient(
diff --git a/tests/services/test_crawler_scope.py b/tests/services/test_crawler_scope.py
new file mode 100644
index 000000000..6ace327d1
--- /dev/null
+++ b/tests/services/test_crawler_scope.py
@@ -0,0 +1,112 @@
+"""Seed-scope semantics of :func:`crawl_docs_site`'s ``base_path_prefix``.
+
+Contract:
+
+- A page seed scopes its parent directory, so sibling pages are internal:
+ ``/intro/home`` seeds ``/intro/*``.
+- A root seed still scopes the whole host.
+- Anything outside the seed's directory — other directories, other hosts,
+ or sibling look-alike prefixes — is rejected.
+- A trailing-slash (directory index) seed stays scoped to its own
+ directory; it must not widen to the whole host.
+"""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+from deeptutor.services.web_source import crawler, robots
+
+
+def crawl_clock(monkeypatch):
+ now = [100.0]
+ monkeypatch.setattr(robots.time, "monotonic", lambda: now[0])
+
+ async def sleep(delay):
+ now[0] += delay
+
+ monkeypatch.setattr(robots.asyncio, "sleep", sleep)
+ monkeypatch.setattr(robots, "_is_disallowed_host", lambda _: False)
+ monkeypatch.setattr(crawler, "_is_disallowed_host", lambda _: False)
+ return now
+
+
+async def _crawl(seed_url: str, page_html: str, requested: list[str]) -> crawler.CrawlResult:
+ """Crawl a mock site that serves *page_html* for every page request.
+
+ Records every requested URL path (``/robots.txt`` included) into
+ *requested*, so tests can assert exactly which pages were fetched.
+ """
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requested.append(request.url.path)
+ if request.url.path == "/robots.txt":
+ return httpx.Response(200, text="User-agent: *\nAllow: /")
+ return httpx.Response(200, text=page_html)
+
+ return await crawler.crawl_docs_site(
+ seed_url,
+ max_depth=1,
+ client_factory=lambda: httpx.AsyncClient(transport=httpx.MockTransport(handler)),
+ )
+
+
+async def test_page_seed_scopes_sibling_directory_and_rejects_outside(crawl_clock):
+ """``/intro/home`` seeds ``/intro/*``; anything else stays out of scope."""
+ requested: list[str] = []
+ result = await _crawl(
+ "https://example.com/intro/home",
+ "<html><body><main>"
+ "<a href='/intro/next'>next</a>"
+ "<a href='/intro'>parent</a>"
+ "<a href='/docs/outside'>outside</a>"
+ "<a href='/intro2/lookalike'>lookalike</a>"
+ "<a href='https://other.example/x'>external</a>"
+ "</main></body></html>",
+ requested,
+ )
+
+ assert result.ok
+ assert not result.errors
+ assert requested[0] == "/robots.txt"
+ assert requested[1] == "/intro/home"
+ # Siblings are internal, including the bare parent directory itself.
+ assert sorted(requested[2:]) == ["/intro", "/intro/next"]
+
+
+async def test_root_seed_still_scopes_whole_host(crawl_clock):
+ """A root URL keeps the historical whole-host scope."""
+ requested: list[str] = []
+ result = await _crawl(
+ "https://example.com/",
+ "<html><body><main><a href='/any/deep'>deep</a></main></body></html>",
+ requested,
+ )
+
+ assert result.ok
+ assert sorted(requested[1:]) == ["/", "/any/deep"]
+
+
+async def test_directory_seed_stays_scoped_to_its_directory(crawl_clock):
+ """A trailing-slash seed is a directory index, not a root override.
+
+ ``Path('/intro/').parent`` is ``/`` — a naive pathlib rewrite would
+ widen a section seed to the entire host.
+ """
+ requested: list[str] = []
+ result = await _crawl(
+ "https://example.com/intro/",
+ "<html><body><main>"
+ "<a href='/intro/other'>other</a>"
+ "<a href='/docs/elsewhere'>elsewhere</a>"
+ "</main></body></html>",
+ requested,
+ )
+
+ assert result.ok
+ assert sorted(requested[1:]) == ["/intro/", "/intro/other"]