Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

feat(index): support .md and .txt files natively in the pipeline (#101)

aafaq-rashid-comprinno55ea7c559431
diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.py
index f943ec8..6d6f618 100644
--- a/index/src/pixelrag_index/pipelines.py
+++ b/index/src/pixelrag_index/pipelines.py
@@ -51,6 +51,7 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:
url_docs = []
pdf_docs = []
image_docs = []
+ text_docs = []
articles = [] # id → metadata mapping for serve
for doc in docs:
@@ -67,6 +68,8 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:
url_docs.append((idx, doc))
elif doc.path and doc.path.lower().endswith(".pdf"):
pdf_docs.append((idx, doc))
+ elif doc.path and (doc.metadata or {}).get("type") == "text":
+ text_docs.append((idx, doc))
elif doc.path:
image_docs.append((idx, doc))
@@ -89,6 +92,46 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:
" Rendered %d URLs (%d skipped, already exist)", len(new_url_docs), skipped
)
+ # Render text files (.md, .txt) — convert to styled HTML, then render via CDP
+ if text_docs:
+ import html as html_mod
+ import tempfile
+
+ _HTML_TEMPLATE = (
+ '<html><head><meta charset="utf-8"><style>'
+ "body { font-family: -apple-system, system-ui, sans-serif; "
+ "max-width: 860px; margin: 40px auto; padding: 20px; line-height: 1.7; "
+ "color: #1a1a1a; } "
+ "pre, code { background: #f4f4f5; padding: 2px 6px; border-radius: 4px; "
+ "font-size: 14px; } "
+ "pre { padding: 16px; overflow-x: auto; white-space: pre-wrap; } "
+ "h1,h2,h3 { color: #111; } "
+ "table { border-collapse: collapse; width: 100%; } "
+ "th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }"
+ "</style></head><body>"
+ )
+ tmp_dir = tempfile.mkdtemp(prefix="pixelrag_text_")
+ text_urls = []
+ text_stems = []
+
+ for idx, doc in text_docs:
+ if (tiles_dir / f"{idx}.png.tiles" / "tiles.json").exists():
+ continue
+ content = Path(doc.path).read_text(errors="replace")
+ escaped = html_mod.escape(content)
+ html_content = f"{_HTML_TEMPLATE}<pre>{escaped}</pre></body></html>"
+ html_path = Path(tmp_dir) / f"{idx}.html"
+ html_path.write_text(html_content)
+ text_urls.append(f"file://{html_path.resolve()}")
+ text_stems.append(str(idx))
+
+ if text_urls:
+ text_ingest = {k: v for k, v in ingest_cfg.items() if k != "backend"}
+ render_urls(
+ text_urls, str(tiles_dir), backend="cdp", stems=text_stems, **text_ingest
+ )
+ logger.info(" Rendered %d text files (.md/.txt)", len(text_docs))
+
# Render PDFs
for idx, doc in pdf_docs:
try:
diff --git a/index/src/pixelrag_index/sources/local.py b/index/src/pixelrag_index/sources/local.py
index e26ee3d..4611fa6 100644
--- a/index/src/pixelrag_index/sources/local.py
+++ b/index/src/pixelrag_index/sources/local.py
@@ -12,6 +12,8 @@ EXTENSIONS = {
".png": "image",
".jpg": "image",
".jpeg": "image",
+ ".md": "text",
+ ".txt": "text",
}
@@ -33,6 +35,12 @@ class LocalSource(Source):
url=f"file://{f.resolve()}",
metadata={"type": ftype},
)
+ elif ftype == "text":
+ yield Document(
+ id=f.stem,
+ path=str(f),
+ metadata={"type": "text", "extension": f.suffix.lower()},
+ )
else:
yield Document(
id=f.stem,
diff --git a/tests/test_local_source_text.py b/tests/test_local_source_text.py
new file mode 100644
index 0000000..160a7be
--- /dev/null
+++ b/tests/test_local_source_text.py
@@ -0,0 +1,58 @@
+"""Tests for native .md/.txt support in the local source adapter (issue #100)."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parents[1] / "index" / "src"))
+from pixelrag_index.sources.local import LocalSource
+
+
+def text_dir(tmp_path):
+ (tmp_path / "guide.md").write_text("# Guide\nSome markdown content")
+ (tmp_path / "notes.txt").write_text("Plain text notes here")
+ (tmp_path / "page.html").write_text("<html><body>HTML</body></html>")
+ (tmp_path / "photo.png").write_bytes(b"\x89PNG\r\n")
+ (tmp_path / "ignored.csv").write_text("a,b,c")
+ return tmp_path
+
+
+def test_local_source_discovers_text_files(text_dir):
+ """LocalSource should find .md and .txt alongside .html and .png."""
+ source = LocalSource(str(text_dir))
+ docs = list(source)
+ names = {d.id for d in docs}
+ assert "guide" in names # .md
+ assert "notes" in names # .txt
+ assert "page" in names # .html
+ assert "photo" in names # .png
+ assert "ignored" not in names # .csv not supported
+
+
+def test_text_files_have_correct_metadata(text_dir):
+ source = LocalSource(str(text_dir))
+ docs = {d.id: d for d in source}
+
+ # Text files should have type=text and a path (not a URL)
+ assert docs["guide"].metadata["type"] == "text"
+ assert docs["guide"].path is not None
+ assert docs["guide"].url is None
+
+ assert docs["notes"].metadata["type"] == "text"
+ assert docs["notes"].path is not None
+
+
+def test_html_files_have_url(text_dir):
+ source = LocalSource(str(text_dir))
+ docs = {d.id: d for d in source}
+
+ # HTML files should have a file:// URL
+ assert docs["page"].url.startswith("file://")
+ assert docs["page"].metadata["type"] == "web"
+
+
+def test_source_len_includes_text_files(text_dir):
+ source = LocalSource(str(text_dir))
+ assert len(source) == 4 # .md + .txt + .html + .png (not .csv)