[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
feat(browser_rag): expose url/domain/visit fields as metadata (#315)
Nathan Kettles152d35a3c0f4
diff --git a/apps/history_data/history.py b/apps/history_data/history.pyindex f7e23c7..db6cab9 100644--- a/apps/history_data/history.py+++ b/apps/history_data/history.py@@ -2,6 +2,7 @@ import osimport sqlite3from pathlib import Pathfrom typing import Any+from urllib.parse import urlparsefrom llama_index.core import Documentfrom llama_index.core.readers.base import BaseReader@@ -77,6 +78,7 @@ class ChromeHistoryReader(BaseReader):last_visit, url, title, visit_count, typed_count, _hidden = row# Create document content with metadata embedded in text+ # (kept in body so semantic search still sees these fields)doc_content = f"""[Title]: {title}[URL of the page]: {url}@@ -85,10 +87,21 @@ class ChromeHistoryReader(BaseReader):[Typed times]: {typed_count}"""- # Create document with embedded metadata- doc = Document(text=doc_content, metadata={"title": title[0:150]})- # if len(title) > 150:- # print(f"Title is too long: {title}")+ # Also expose structured fields via metadata so they can be used+ # with metadata_filters at search time (see docs/metadata_filtering.md).+ domain = urlparse(url).netloc if url else ""++ doc = Document(+ text=doc_content,+ metadata={+ "title": (title or "")[0:150],+ "url": url or "",+ "domain": domain,+ "last_visited": str(last_visit) if last_visit is not None else "",+ "visit_count": int(visit_count) if visit_count is not None else 0,+ "typed_count": int(typed_count) if typed_count is not None else 0,+ },+ )docs.append(doc)count += 1diff --git a/packages/leann-core/src/leann/chunking_utils.py b/packages/leann-core/src/leann/chunking_utils.pyindex b220007..965828a 100644--- a/packages/leann-core/src/leann/chunking_utils.py+++ b/packages/leann-core/src/leann/chunking_utils.py@@ -312,16 +312,12 @@ def create_traditional_chunks(result = []for doc in documents:- # Extract document-level metadata- doc_metadata = {- "file_path": doc.metadata.get("file_path", ""),- "file_name": doc.metadata.get("file_name", ""),- "source": doc.metadata.get("source", ""),- }- if "creation_date" in doc.metadata:- doc_metadata["creation_date"] = doc.metadata["creation_date"]- if "last_modified_date" in doc.metadata:- doc_metadata["last_modified_date"] = doc.metadata["last_modified_date"]+ # Propagate all document-level metadata to each chunk so custom fields+ # (e.g. url/domain for browser_rag) remain available for metadata_filters.+ doc_metadata = dict(doc.metadata) if doc.metadata else {}+ doc_metadata.setdefault("file_path", "")+ doc_metadata.setdefault("file_name", "")+ doc_metadata.setdefault("source", "")try:nodes = node_parser.get_nodes_from_documents([doc])