Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

feat: add Gemini CLI and Qwen Code RAG integration (#218)

Arshad Jafria72320c252e9
diff --git a/apps/gemini_data/__init__.py b/apps/gemini_data/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/apps/gemini_data/gemini_reader.py b/apps/gemini_data/gemini_reader.py
new file mode 100644
index 0000000..a4e23b8
--- /dev/null
+++ b/apps/gemini_data/gemini_reader.py
@@ -0,0 +1,149 @@
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+class GeminiReader:
+ """Reader for Gemini CLI history files."""
+
+ def __init__(self):
+ pass
+
+ def load_data(self, history_dir: str, max_count: int = -1) -> list[dict[str, Any]]:
+ """
+ Load data from Gemini history directory.
+
+ Args:
+ history_dir: Path to .gemini directory
+ max_count: Max number of conversations to load
+
+ Returns:
+ List of dictionaries with 'text' and 'metadata' keys
+ """
+ history_path = Path(history_dir).expanduser()
+ if not history_path.exists():
+ print(f"Gemini history directory not found: {history_path}")
+ return []
+
+ documents = []
+
+ # 1. Load Memory (GEMINI.md)
+ memory_file = history_path / "GEMINI.md"
+ if memory_file.exists():
+ try:
+ text = memory_file.read_text(encoding="utf-8")
+ if text.strip():
+ documents.append(
+ {
+ "text": f"Gemini Memory:\n{text}",
+ "metadata": {"source": str(memory_file), "type": "memory"},
+ }
+ )
+ except Exception as e:
+ print(f"Error reading memory file: {e}")
+
+ # 2. Find Session Files
+ # Legacy JSON sessions
+ session_files = list(history_path.glob("session-*.json"))
+ # New JSONL sessions
+ session_files.extend(list(history_path.glob("session-*.jsonl")))
+ # Checkpoints
+ session_files.extend(list(history_path.glob("checkpoint-*.json")))
+
+ # Sort by modification time (newest first)
+ session_files.sort(key=lambda x: x.stat().st_mtime, reverse=True)
+
+ print(f"Found {len(session_files)} session files.")
+
+ count = 0
+ for file_path in session_files:
+ if max_count > 0 and count >= max_count:
+ break
+
+ try:
+ content = ""
+ if file_path.suffix == ".jsonl":
+ content = self._parse_jsonl_session(file_path)
+ elif file_path.suffix == ".json":
+ content = self._parse_json_session(file_path)
+
+ if content:
+ documents.append(
+ {
+ "text": content,
+ "metadata": {
+ "source": str(file_path),
+ "type": "session",
+ "filename": file_path.name,
+ },
+ }
+ )
+ count += 1
+ except Exception as e:
+ print(f"Error reading {file_path.name}: {e}")
+
+ print(f"Successfully loaded {len(documents)} items from Gemini history.")
+ return documents
+
+ def _parse_json_session(self, file_path: Path) -> str:
+ """Parse legacy JSON session file."""
+ data = json.loads(file_path.read_text(encoding="utf-8"))
+
+ # Handle dict format (standard session)
+ messages = []
+ if isinstance(data, dict):
+ # Check for 'messages' key (standard format)
+ if "messages" in data:
+ for msg in data["messages"]:
+ role = msg.get("role", "unknown")
+ content = msg.get("content", "")
+ if content:
+ messages.append(f"{role.upper()}: {content}")
+ # Check for 'parts' key (checkpoint format sometimes)
+ elif "parts" in data:
+ messages.append(f"Saved Session Content: {data['parts']}")
+
+ # Handle list format (some older array-based sessions)
+ elif isinstance(data, list):
+ for item in data:
+ if isinstance(item, dict):
+ role = item.get("role", "unknown")
+ content = item.get("content", "") or item.get("parts", "")
+ if content:
+ messages.append(f"{role.upper()}: {content}")
+
+ if not messages:
+ return ""
+
+ return f"File: {file_path.name}\n\n" + "\n\n".join(messages)
+
+ def _parse_jsonl_session(self, file_path: Path) -> str:
+ """Parse JSONL session file."""
+ messages = []
+ try:
+ with open(file_path, encoding="utf-8") as f:
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ data = json.loads(line)
+ # Skip metadata lines if they don't have content
+ if "role" in data and "content" in data:
+ messages.append(f"{data['role'].upper()}: {data['content']}")
+ elif "parts" in data: # sometimes parts is used
+ messages.append(
+ f"{data.get('role', 'unknown').upper()}: {data['parts']}"
+ )
+ except json.JSONDecodeError:
+ continue
+ except Exception:
+ return ""
+
+ if not messages:
+ return ""
+
+ return f"File: {file_path.name}\n\n" + "\n\n".join(messages)
diff --git a/apps/gemini_rag.py b/apps/gemini_rag.py
new file mode 100644
index 0000000..9578850
--- /dev/null
+++ b/apps/gemini_rag.py
@@ -0,0 +1,69 @@
+"""
+Gemini CLI RAG example.
+Indexes and searches Gemini CLI history (~/.gemini).
+"""
+
+import sys
+from pathlib import Path
+from typing import Any
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+from base_rag_example import BaseRAGExample
+from chunking import create_text_chunks
+
+from .gemini_data.gemini_reader import GeminiReader
+
+
+class GeminiRAG(BaseRAGExample):
+ """RAG example for Gemini CLI history."""
+
+ def __init__(self):
+ super().__init__(
+ name="Gemini CLI",
+ description="Process and query Gemini CLI history with LEANN",
+ default_index_name="gemini_index",
+ )
+
+ def _add_specific_arguments(self, parser):
+ """Add Gemini-specific arguments."""
+ group = parser.add_argument_group("Gemini Parameters")
+ group.add_argument(
+ "--gemini-path",
+ type=str,
+ default="~/.gemini",
+ help="Path to .gemini directory (default: ~/.gemini)",
+ )
+
+ async def load_data(self, args) -> list[dict[str, Any]]:
+ """Load Gemini history and convert to text chunks."""
+ print(f"Loading Gemini history from: {args.gemini_path}")
+
+ reader = GeminiReader()
+ documents = reader.load_data(history_dir=args.gemini_path, max_count=args.max_items)
+
+ if not documents:
+ print("No documents found! Check if ~/.gemini exists and has history.")
+ return []
+
+ # Convert dicts to Document objects for chunking
+ from llama_index.core import Document
+
+ docs = [Document(text=d["text"], metadata=d["metadata"]) for d in documents]
+
+ # Convert to text chunks
+ print(f"splitting {len(documents)} documents into chunks...")
+ chunks = create_text_chunks(docs)
+
+ return chunks
+
+
+if __name__ == "__main__":
+ import asyncio
+
+ print("\n✨ Gemini CLI RAG")
+ print("=" * 50)
+
+ rag = GeminiRAG()
+ asyncio.run(rag.run())
diff --git a/apps/qwen_data/__init__.py b/apps/qwen_data/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/apps/qwen_data/qwen_reader.py b/apps/qwen_data/qwen_reader.py
new file mode 100644
index 0000000..865121d
--- /dev/null
+++ b/apps/qwen_data/qwen_reader.py
@@ -0,0 +1,150 @@
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+class QwenReader:
+ """Reader for Qwen Code CLI history files."""
+
+ def __init__(self):
+ pass
+
+ def load_data(self, history_dir: str, max_count: int = -1) -> list[dict[str, Any]]:
+ """
+ Load data from Qwen Code history directory.
+
+ Args:
+ history_dir: Path to .qwen-code directory
+ max_count: Max number of conversations to load
+
+ Returns:
+ List of dictionaries with 'text' and 'metadata' keys
+ """
+ history_path = Path(history_dir).expanduser()
+ if not history_path.exists():
+ print(f"Qwen history directory not found: {history_path}")
+ return []
+
+ documents = []
+
+ # 1. Load Memory (QWEN.md or MEMORY.md)
+ for memory_filename in ["QWEN.md", "MEMORY.md"]:
+ memory_file = history_path / memory_filename
+ if memory_file.exists():
+ try:
+ text = memory_file.read_text(encoding="utf-8")
+ if text.strip():
+ documents.append(
+ {
+ "text": f"Qwen Code Memory:\n{text}",
+ "metadata": {"source": str(memory_file), "type": "memory"},
+ }
+ )
+ except Exception as e:
+ print(f"Error reading memory file {memory_filename}: {e}")
+
+ # 2. Find Session Files
+ # Legacy JSON sessions
+ session_files = list(history_path.glob("session-*.json"))
+ # New JSONL sessions
+ session_files.extend(list(history_path.glob("session-*.jsonl")))
+ # Checkpoints
+ session_files.extend(list(history_path.glob("checkpoint-*.json")))
+
+ # Sort by modification time (newest first)
+ session_files.sort(key=lambda x: x.stat().st_mtime, reverse=True)
+
+ print(f"Found {len(session_files)} session files.")
+
+ count = 0
+ for file_path in session_files:
+ if max_count > 0 and count >= max_count:
+ break
+
+ try:
+ content = ""
+ if file_path.suffix == ".jsonl":
+ content = self._parse_jsonl_session(file_path)
+ elif file_path.suffix == ".json":
+ content = self._parse_json_session(file_path)
+
+ if content:
+ documents.append(
+ {
+ "text": content,
+ "metadata": {
+ "source": str(file_path),
+ "type": "session",
+ "filename": file_path.name,
+ },
+ }
+ )
+ count += 1
+ except Exception as e:
+ print(f"Error reading {file_path.name}: {e}")
+
+ print(f"Successfully loaded {len(documents)} items from Qwen history.")
+ return documents
+
+ def _parse_json_session(self, file_path: Path) -> str:
+ """Parse legacy JSON session file."""
+ data = json.loads(file_path.read_text(encoding="utf-8"))
+
+ # Handle dict format (standard session)
+ messages = []
+ if isinstance(data, dict):
+ # Check for 'messages' key (standard format)
+ if "messages" in data:
+ for msg in data["messages"]:
+ role = msg.get("role", "unknown")
+ content = msg.get("content", "")
+ if content:
+ messages.append(f"{role.upper()}: {content}")
+ # Check for 'parts' key (checkpoint format sometimes)
+ elif "parts" in data:
+ messages.append(f"Saved Session Content: {data['parts']}")
+
+ # Handle list format (some older array-based sessions)
+ elif isinstance(data, list):
+ for item in data:
+ if isinstance(item, dict):
+ role = item.get("role", "unknown")
+ content = item.get("content", "") or item.get("parts", "")
+ if content:
+ messages.append(f"{role.upper()}: {content}")
+
+ if not messages:
+ return ""
+
+ return f"File: {file_path.name}\n\n" + "\n\n".join(messages)
+
+ def _parse_jsonl_session(self, file_path: Path) -> str:
+ """Parse JSONL session file."""
+ messages = []
+ try:
+ with open(file_path, encoding="utf-8") as f:
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ data = json.loads(line)
+ # Skip metadata lines if they don't have content
+ if "role" in data and "content" in data:
+ messages.append(f"{data['role'].upper()}: {data['content']}")
+ elif "parts" in data: # sometimes parts is used
+ messages.append(
+ f"{data.get('role', 'unknown').upper()}: {data['parts']}"
+ )
+ except json.JSONDecodeError:
+ continue
+ except Exception:
+ return ""
+
+ if not messages:
+ return ""
+
+ return f"File: {file_path.name}\n\n" + "\n\n".join(messages)
diff --git a/apps/qwen_rag.py b/apps/qwen_rag.py
new file mode 100644
index 0000000..ae665be
--- /dev/null
+++ b/apps/qwen_rag.py
@@ -0,0 +1,69 @@
+"""
+Qwen Code RAG example.
+Indexes and searches Qwen Code CLI history (~/.qwen-code).
+"""
+
+import sys
+from pathlib import Path
+from typing import Any
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent))
+
+from base_rag_example import BaseRAGExample
+from chunking import create_text_chunks
+
+from .qwen_data.qwen_reader import QwenReader
+
+
+class QwenRAG(BaseRAGExample):
+ """RAG example for Qwen Code CLI history."""
+
+ def __init__(self):
+ super().__init__(
+ name="Qwen Code",
+ description="Process and query Qwen Code CLI history with LEANN",
+ default_index_name="qwen_index",
+ )
+
+ def _add_specific_arguments(self, parser):
+ """Add Qwen-specific arguments."""
+ group = parser.add_argument_group("Qwen Parameters")
+ group.add_argument(
+ "--qwen-path",
+ type=str,
+ default="~/.qwen-code",
+ help="Path to .qwen-code directory (default: ~/.qwen-code)",
+ )
+
+ async def load_data(self, args) -> list[dict[str, Any]]:
+ """Load Qwen history and convert to text chunks."""
+ print(f"Loading Qwen history from: {args.qwen_path}")
+
+ reader = QwenReader()
+ documents = reader.load_data(history_dir=args.qwen_path, max_count=args.max_items)
+
+ if not documents:
+ print("No documents found! Check if ~/.qwen-code exists and has history.")
+ return []
+
+ # Convert dicts to Document objects for chunking
+ from llama_index.core import Document
+
+ docs = [Document(text=d["text"], metadata=d["metadata"]) for d in documents]
+
+ # Convert to text chunks
+ print(f"splitting {len(documents)} documents into chunks...")
+ chunks = create_text_chunks(docs)
+
+ return chunks
+
+
+if __name__ == "__main__":
+ import asyncio
+
+ print("\n✨ Qwen Code RAG")
+ print("=" * 50)
+
+ rag = QwenRAG()
+ asyncio.run(rag.run())