DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(web-source): scope crawl seeds to their parent directory
XZH482d62a0f65c
diff --git a/deeptutor/services/web_source/crawler.py b/deeptutor/services/web_source/crawler.pyindex f00f9f4de..1e827ca56 100644--- a/deeptutor/services/web_source/crawler.py+++ b/deeptutor/services/web_source/crawler.py@@ -381,7 +381,17 @@ async def crawl_docs_site(result.errors.append(f"Disallowed host: {base_host}")return result- base_path_prefix = parsed.path or "/"+ # Scope the crawl to the seed's directory so sibling pages count as+ # internal: /intro/home seeds /intro/*, a root URL seeds the whole host.+ # Pure URL-path rules (not pathlib) — Path("/intro/").parent is "/", which+ # would silently widen a section seed to the entire host, and WindowsPath+ # renders separators as "\", breaking the prefix entirely.+ raw_path = parsed.path or "/"+ if raw_path.endswith("/"):+ # Directory index (or root): the directory itself is the scope.+ base_path_prefix = raw_path+ else:+ base_path_prefix = raw_path.rsplit("/", 1)[0] or "/"factory = client_factory or (lambda: httpx.AsyncClient(diff --git a/tests/services/test_crawler_scope.py b/tests/services/test_crawler_scope.pynew file mode 100644index 000000000..6ace327d1--- /dev/null+++ b/tests/services/test_crawler_scope.py@@ -0,0 +1,112 @@+"""Seed-scope semantics of :func:`crawl_docs_site`'s ``base_path_prefix``.++Contract:++- A page seed scopes its parent directory, so sibling pages are internal:+ ``/intro/home`` seeds ``/intro/*``.+- A root seed still scopes the whole host.+- Anything outside the seed's directory — other directories, other hosts,+ or sibling look-alike prefixes — is rejected.+- A trailing-slash (directory index) seed stays scoped to its own+ directory; it must not widen to the whole host.+"""++from __future__ import annotations++import httpx+import pytest++from deeptutor.services.web_source import crawler, robots+++def crawl_clock(monkeypatch):+ now = [100.0]+ monkeypatch.setattr(robots.time, "monotonic", lambda: now[0])++ async def sleep(delay):+ now[0] += delay++ monkeypatch.setattr(robots.asyncio, "sleep", sleep)+ monkeypatch.setattr(robots, "_is_disallowed_host", lambda _: False)+ monkeypatch.setattr(crawler, "_is_disallowed_host", lambda _: False)+ return now+++async def _crawl(seed_url: str, page_html: str, requested: list[str]) -> crawler.CrawlResult:+ """Crawl a mock site that serves *page_html* for every page request.++ Records every requested URL path (``/robots.txt`` included) into+ *requested*, so tests can assert exactly which pages were fetched.+ """++ def handler(request: httpx.Request) -> httpx.Response:+ requested.append(request.url.path)+ if request.url.path == "/robots.txt":+ return httpx.Response(200, text="User-agent: *\nAllow: /")+ return httpx.Response(200, text=page_html)++ return await crawler.crawl_docs_site(+ seed_url,+ max_depth=1,+ client_factory=lambda: httpx.AsyncClient(transport=httpx.MockTransport(handler)),+ )+++async def test_page_seed_scopes_sibling_directory_and_rejects_outside(crawl_clock):+ """``/intro/home`` seeds ``/intro/*``; anything else stays out of scope."""+ requested: list[str] = []+ result = await _crawl(+ "https://example.com/intro/home",+ "<html><body><main>"+ "<a href='/intro/next'>next</a>"+ "<a href='/intro'>parent</a>"+ "<a href='/docs/outside'>outside</a>"+ "<a href='/intro2/lookalike'>lookalike</a>"+ "<a href='https://other.example/x'>external</a>"+ "</main></body></html>",+ requested,+ )++ assert result.ok+ assert not result.errors+ assert requested[0] == "/robots.txt"+ assert requested[1] == "/intro/home"+ # Siblings are internal, including the bare parent directory itself.+ assert sorted(requested[2:]) == ["/intro", "/intro/next"]+++async def test_root_seed_still_scopes_whole_host(crawl_clock):+ """A root URL keeps the historical whole-host scope."""+ requested: list[str] = []+ result = await _crawl(+ "https://example.com/",+ "<html><body><main><a href='/any/deep'>deep</a></main></body></html>",+ requested,+ )++ assert result.ok+ assert sorted(requested[1:]) == ["/", "/any/deep"]+++async def test_directory_seed_stays_scoped_to_its_directory(crawl_clock):+ """A trailing-slash seed is a directory index, not a root override.++ ``Path('/intro/').parent`` is ``/`` — a naive pathlib rewrite would+ widen a section seed to the entire host.+ """+ requested: list[str] = []+ result = await _crawl(+ "https://example.com/intro/",+ "<html><body><main>"+ "<a href='/intro/other'>other</a>"+ "<a href='/docs/elsewhere'>elsewhere</a>"+ "</main></body></html>",+ requested,+ )++ assert result.ok+ assert sorted(requested[1:]) == ["/intro/", "/intro/other"]