Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

fix(index): skip indented comment lines in WebSource url files (#169)

Zhibo Zhang (Amur)c47b4c52389d
diff --git a/index/src/pixelrag_index/sources/web.py b/index/src/pixelrag_index/sources/web.py
index 87b9fd1..8a5b215 100644
--- a/index/src/pixelrag_index/sources/web.py
+++ b/index/src/pixelrag_index/sources/web.py
@@ -69,9 +69,9 @@ class WebSource(Source):
if p.exists():
with open(p, encoding="utf-8") as f:
self._urls = [
- line.strip()
+ cleaned
for line in f
- if line.strip() and not line.startswith("#")
+ if (cleaned := line.strip()) and not cleaned.startswith("#")
]
else:
raise FileNotFoundError(f"urls_file not found: {urls_file}")
diff --git a/tests/test_web_source_urls.py b/tests/test_web_source_urls.py
new file mode 100644
index 0000000..8a5b29a
--- /dev/null
+++ b/tests/test_web_source_urls.py
@@ -0,0 +1,34 @@
+"""Regression tests for WebSource URL file parsing."""
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parents[1] / "index" / "src"))
+
+from pixelrag_index.sources.web import WebSource
+
+
+def _write(tmp_path, content):
+ p = tmp_path / "urls.txt"
+ p.write_text(content, encoding="utf-8")
+ return WebSource(urls_file=str(p))
+
+
+def test_plain_urls_are_kept(tmp_path):
+ src = _write(tmp_path, "https://a.com\nhttps://b.com\n")
+ assert src._urls == ["https://a.com", "https://b.com"]
+
+
+def test_indented_comment_lines_are_skipped(tmp_path):
+ src = _write(tmp_path, "https://a.com\n # indented comment\n\thttps://b.com\n")
+ assert src._urls == ["https://a.com", "https://b.com"]
+
+
+def test_blank_and_full_comment_lines_are_skipped(tmp_path):
+ src = _write(tmp_path, "# top comment\n\nhttps://a.com\n \n# another\n")
+ assert src._urls == ["https://a.com"]
+
+
+def test_surrounding_whitespace_is_stripped(tmp_path):
+ src = _write(tmp_path, " https://a.com \n")
+ assert src._urls == ["https://a.com"]