https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
feat(index): support .md and .txt files natively in the pipeline (#101)
aafaq-rashid-comprinno55ea7c559431
diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.pyindex f943ec8..6d6f618 100644--- a/index/src/pixelrag_index/pipelines.py+++ b/index/src/pixelrag_index/pipelines.py@@ -51,6 +51,7 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:url_docs = []pdf_docs = []image_docs = []+ text_docs = []articles = [] # id → metadata mapping for servefor doc in docs:@@ -67,6 +68,8 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:url_docs.append((idx, doc))elif doc.path and doc.path.lower().endswith(".pdf"):pdf_docs.append((idx, doc))+ elif doc.path and (doc.metadata or {}).get("type") == "text":+ text_docs.append((idx, doc))elif doc.path:image_docs.append((idx, doc))@@ -89,6 +92,46 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:" Rendered %d URLs (%d skipped, already exist)", len(new_url_docs), skipped)+ # Render text files (.md, .txt) — convert to styled HTML, then render via CDP+ if text_docs:+ import html as html_mod+ import tempfile++ _HTML_TEMPLATE = (+ '<html><head><meta charset="utf-8"><style>'+ "body { font-family: -apple-system, system-ui, sans-serif; "+ "max-width: 860px; margin: 40px auto; padding: 20px; line-height: 1.7; "+ "color: #1a1a1a; } "+ "pre, code { background: #f4f4f5; padding: 2px 6px; border-radius: 4px; "+ "font-size: 14px; } "+ "pre { padding: 16px; overflow-x: auto; white-space: pre-wrap; } "+ "h1,h2,h3 { color: #111; } "+ "table { border-collapse: collapse; width: 100%; } "+ "th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }"+ "</style></head><body>"+ )+ tmp_dir = tempfile.mkdtemp(prefix="pixelrag_text_")+ text_urls = []+ text_stems = []++ for idx, doc in text_docs:+ if (tiles_dir / f"{idx}.png.tiles" / "tiles.json").exists():+ continue+ content = Path(doc.path).read_text(errors="replace")+ escaped = html_mod.escape(content)+ html_content = f"{_HTML_TEMPLATE}<pre>{escaped}</pre></body></html>"+ html_path = Path(tmp_dir) / f"{idx}.html"+ html_path.write_text(html_content)+ text_urls.append(f"file://{html_path.resolve()}")+ text_stems.append(str(idx))++ if text_urls:+ text_ingest = {k: v for k, v in ingest_cfg.items() if k != "backend"}+ render_urls(+ text_urls, str(tiles_dir), backend="cdp", stems=text_stems, **text_ingest+ )+ logger.info(" Rendered %d text files (.md/.txt)", len(text_docs))+# Render PDFsfor idx, doc in pdf_docs:try:diff --git a/index/src/pixelrag_index/sources/local.py b/index/src/pixelrag_index/sources/local.pyindex e26ee3d..4611fa6 100644--- a/index/src/pixelrag_index/sources/local.py+++ b/index/src/pixelrag_index/sources/local.py@@ -12,6 +12,8 @@ EXTENSIONS = {".png": "image",".jpg": "image",".jpeg": "image",+ ".md": "text",+ ".txt": "text",}@@ -33,6 +35,12 @@ class LocalSource(Source):url=f"file://{f.resolve()}",metadata={"type": ftype},)+ elif ftype == "text":+ yield Document(+ id=f.stem,+ path=str(f),+ metadata={"type": "text", "extension": f.suffix.lower()},+ )else:yield Document(id=f.stem,diff --git a/tests/test_local_source_text.py b/tests/test_local_source_text.pynew file mode 100644index 0000000..160a7be--- /dev/null+++ b/tests/test_local_source_text.py@@ -0,0 +1,58 @@+"""Tests for native .md/.txt support in the local source adapter (issue #100)."""++import sys+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).parents[1] / "index" / "src"))+from pixelrag_index.sources.local import LocalSource+++def text_dir(tmp_path):+ (tmp_path / "guide.md").write_text("# Guide\nSome markdown content")+ (tmp_path / "notes.txt").write_text("Plain text notes here")+ (tmp_path / "page.html").write_text("<html><body>HTML</body></html>")+ (tmp_path / "photo.png").write_bytes(b"\x89PNG\r\n")+ (tmp_path / "ignored.csv").write_text("a,b,c")+ return tmp_path+++def test_local_source_discovers_text_files(text_dir):+ """LocalSource should find .md and .txt alongside .html and .png."""+ source = LocalSource(str(text_dir))+ docs = list(source)+ names = {d.id for d in docs}+ assert "guide" in names # .md+ assert "notes" in names # .txt+ assert "page" in names # .html+ assert "photo" in names # .png+ assert "ignored" not in names # .csv not supported+++def test_text_files_have_correct_metadata(text_dir):+ source = LocalSource(str(text_dir))+ docs = {d.id: d for d in source}++ # Text files should have type=text and a path (not a URL)+ assert docs["guide"].metadata["type"] == "text"+ assert docs["guide"].path is not None+ assert docs["guide"].url is None++ assert docs["notes"].metadata["type"] == "text"+ assert docs["notes"].path is not None+++def test_html_files_have_url(text_dir):+ source = LocalSource(str(text_dir))+ docs = {d.id: d for d in source}++ # HTML files should have a file:// URL+ assert docs["page"].url.startswith("file://")+ assert docs["page"].metadata["type"] == "web"+++def test_source_len_includes_text_files(text_dir):+ source = LocalSource(str(text_dir))+ assert len(source) == 4 # .md + .txt + .html + .png (not .csv)