~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Custom YAML document loader.
LangChain doesn't have a dedicated YAML loader, so we provide one here.
This loader reads YAML files and converts them to readable text documents.
"""
from pathlib import Path
from typing import Iterator
import yaml # type: ignore[import-untyped]
from langchain_core.document_loaders import BaseLoader
from langchain_core.documents import Document
from loguru import logger
from .text_encoding import _file_encoding
class YAMLLoader(BaseLoader):
"""
Load YAML files and convert to LangChain Documents.
The YAML content is rendered as formatted YAML text for human readability
and semantic search. Complex structures are preserved in their YAML format.
"""
def __init__(self, file_path: str | Path, encoding: str | None = None):
"""
Initialize the YAML loader.
Args:
file_path: Path to the YAML file
encoding: File encoding (default: detected, as for text uploads)
"""
self.file_path = Path(file_path)
self.encoding = encoding
def lazy_load(self) -> Iterator[Document]:
"""
Lazily load documents from the YAML file.
Yields:
Document objects containing the YAML content as text
"""
try:
# Detected like a text upload's: a byte-order mark is dropped, and UTF-16
# from PowerShell or Windows-1252 from Notepad reads as text
encoding = self.encoding or _file_encoding(self.file_path)
# Recorded as for text uploads, when it was detected
detected = {} if self.encoding else {"detected_encoding": encoding}
with open(self.file_path, encoding=encoding) as f:
content = f.read()
# Parse YAML to validate and normalize
try:
data = yaml.safe_load(content)
except yaml.YAMLError:
logger.exception(f"Invalid YAML in {self.file_path}")
# Still yield the raw content if parsing fails
yield Document(
page_content=content,
metadata={
"source": str(self.file_path),
"parse_error": True,
**detected,
},
)
return
# Skip empty YAML documents
if data is None:
logger.debug(f"Skipping empty YAML document: {self.file_path}")
return
# Convert back to formatted YAML for readability
# Use default_flow_style=False for block-style output
text = yaml.dump(
data,
default_flow_style=False,
allow_unicode=True,
sort_keys=False,
)
yield Document(
page_content=text,
metadata={
"source": str(self.file_path),
"file_type": "yaml",
**detected,
},
)
except Exception:
logger.exception(f"Error loading YAML file: {self.file_path}")
raise
def load(self) -> list[Document]:
"""Load all documents from the YAML file."""
return list(self.lazy_load())
def extract_text_from_yaml(content: bytes, encoding: str = "utf-8") -> str:
"""
Extract text from YAML bytes content.
This is a convenience function for use with bytes (e.g., from file uploads).
Args:
content: YAML content as bytes
encoding: Text encoding (default: utf-8)
Returns:
Formatted YAML text
"""
try:
text = content.decode(encoding)
except UnicodeDecodeError:
text = content.decode(encoding, errors="ignore")
logger.warning(
"YAML content had encoding errors, some characters may be lost"
)
try:
data = yaml.safe_load(text)
# Skip empty YAML documents
if data is None:
return ""
return yaml.dump( # type: ignore[no-any-return]
data,
default_flow_style=False,
allow_unicode=True,
sort_keys=False,
)
except yaml.YAMLError:
logger.exception("Invalid YAML content")
# Return raw text if parsing fails
return text