Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

1.6 KB
Raw Blame
"""Local source — a single file or a directory tree; auto-detect file types."""

from pathlib import Path
from typing import Iterator

from .base import Document, Source

EXTENSIONS = {
    ".pdf": "pdf",
    ".html": "web",
    ".htm": "web",
    ".png": "image",
    ".jpg": "image",
    ".jpeg": "image",
    ".md": "text",
    ".txt": "text",
}


class LocalSource(Source):
    def __init__(self, path: str, **kwargs):
        self.path = Path(path)
        # A path pointing straight at a document is a single-item source; rglob
        # on a file yields nothing, so branch before walking.
        candidates = (
            [self.path] if self.path.is_file() else sorted(self.path.rglob("*"))
        )
        self._files = [
            f for f in candidates if f.is_file() and f.suffix.lower() in EXTENSIONS
        ]

    def __iter__(self) -> Iterator[Document]:
        for f in self._files:
            ftype = EXTENSIONS.get(f.suffix.lower(), "unknown")
            if ftype == "web":
                yield Document(
                    id=f.stem,
                    url=f.resolve().as_uri(),
                    metadata={"type": ftype},
                )
            elif ftype == "text":
                yield Document(
                    id=f.stem,
                    path=str(f),
                    metadata={"type": "text", "extension": f.suffix.lower()},
                )
            else:
                yield Document(
                    id=f.stem,
                    path=str(f),
                    metadata={"type": ftype},
                )

    def __len__(self) -> int:
        return len(self._files)