Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

feat(browser_rag): expose url/domain/visit fields as metadata (#315)

Nathan Kettles152d35a3c0f4
diff --git a/apps/history_data/history.py b/apps/history_data/history.py
index f7e23c7..db6cab9 100644
--- a/apps/history_data/history.py
+++ b/apps/history_data/history.py
@@ -2,6 +2,7 @@ import os
import sqlite3
from pathlib import Path
from typing import Any
+from urllib.parse import urlparse
from llama_index.core import Document
from llama_index.core.readers.base import BaseReader
@@ -77,6 +78,7 @@ class ChromeHistoryReader(BaseReader):
last_visit, url, title, visit_count, typed_count, _hidden = row
# Create document content with metadata embedded in text
+ # (kept in body so semantic search still sees these fields)
doc_content = f"""
[Title]: {title}
[URL of the page]: {url}
@@ -85,10 +87,21 @@ class ChromeHistoryReader(BaseReader):
[Typed times]: {typed_count}
"""
- # Create document with embedded metadata
- doc = Document(text=doc_content, metadata={"title": title[0:150]})
- # if len(title) > 150:
- # print(f"Title is too long: {title}")
+ # Also expose structured fields via metadata so they can be used
+ # with metadata_filters at search time (see docs/metadata_filtering.md).
+ domain = urlparse(url).netloc if url else ""
+
+ doc = Document(
+ text=doc_content,
+ metadata={
+ "title": (title or "")[0:150],
+ "url": url or "",
+ "domain": domain,
+ "last_visited": str(last_visit) if last_visit is not None else "",
+ "visit_count": int(visit_count) if visit_count is not None else 0,
+ "typed_count": int(typed_count) if typed_count is not None else 0,
+ },
+ )
docs.append(doc)
count += 1
diff --git a/packages/leann-core/src/leann/chunking_utils.py b/packages/leann-core/src/leann/chunking_utils.py
index b220007..965828a 100644
--- a/packages/leann-core/src/leann/chunking_utils.py
+++ b/packages/leann-core/src/leann/chunking_utils.py
@@ -312,16 +312,12 @@ def create_traditional_chunks(
result = []
for doc in documents:
- # Extract document-level metadata
- doc_metadata = {
- "file_path": doc.metadata.get("file_path", ""),
- "file_name": doc.metadata.get("file_name", ""),
- "source": doc.metadata.get("source", ""),
- }
- if "creation_date" in doc.metadata:
- doc_metadata["creation_date"] = doc.metadata["creation_date"]
- if "last_modified_date" in doc.metadata:
- doc_metadata["last_modified_date"] = doc.metadata["last_modified_date"]
+ # Propagate all document-level metadata to each chunk so custom fields
+ # (e.g. url/domain for browser_rag) remain available for metadata_filters.
+ doc_metadata = dict(doc.metadata) if doc.metadata else {}
+ doc_metadata.setdefault("file_path", "")
+ doc_metadata.setdefault("file_name", "")
+ doc_metadata.setdefault("source", "")
try:
nodes = node_parser.get_nodes_from_documents([doc])