[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
feat: add Gemini CLI and Qwen Code RAG integration (#218)
Arshad Jafria72320c252e9
diff --git a/apps/gemini_data/__init__.py b/apps/gemini_data/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/apps/gemini_data/gemini_reader.py b/apps/gemini_data/gemini_reader.pynew file mode 100644index 0000000..a4e23b8--- /dev/null+++ b/apps/gemini_data/gemini_reader.py@@ -0,0 +1,149 @@+import json+import logging+from pathlib import Path+from typing import Any++logger = logging.getLogger(__name__)+++class GeminiReader:+ """Reader for Gemini CLI history files."""++ def __init__(self):+ pass++ def load_data(self, history_dir: str, max_count: int = -1) -> list[dict[str, Any]]:+ """+ Load data from Gemini history directory.++ Args:+ history_dir: Path to .gemini directory+ max_count: Max number of conversations to load++ Returns:+ List of dictionaries with 'text' and 'metadata' keys+ """+ history_path = Path(history_dir).expanduser()+ if not history_path.exists():+ print(f"Gemini history directory not found: {history_path}")+ return []++ documents = []++ # 1. Load Memory (GEMINI.md)+ memory_file = history_path / "GEMINI.md"+ if memory_file.exists():+ try:+ text = memory_file.read_text(encoding="utf-8")+ if text.strip():+ documents.append(+ {+ "text": f"Gemini Memory:\n{text}",+ "metadata": {"source": str(memory_file), "type": "memory"},+ }+ )+ except Exception as e:+ print(f"Error reading memory file: {e}")++ # 2. Find Session Files+ # Legacy JSON sessions+ session_files = list(history_path.glob("session-*.json"))+ # New JSONL sessions+ session_files.extend(list(history_path.glob("session-*.jsonl")))+ # Checkpoints+ session_files.extend(list(history_path.glob("checkpoint-*.json")))++ # Sort by modification time (newest first)+ session_files.sort(key=lambda x: x.stat().st_mtime, reverse=True)++ print(f"Found {len(session_files)} session files.")++ count = 0+ for file_path in session_files:+ if max_count > 0 and count >= max_count:+ break++ try:+ content = ""+ if file_path.suffix == ".jsonl":+ content = self._parse_jsonl_session(file_path)+ elif file_path.suffix == ".json":+ content = self._parse_json_session(file_path)++ if content:+ documents.append(+ {+ "text": content,+ "metadata": {+ "source": str(file_path),+ "type": "session",+ "filename": file_path.name,+ },+ }+ )+ count += 1+ except Exception as e:+ print(f"Error reading {file_path.name}: {e}")++ print(f"Successfully loaded {len(documents)} items from Gemini history.")+ return documents++ def _parse_json_session(self, file_path: Path) -> str:+ """Parse legacy JSON session file."""+ data = json.loads(file_path.read_text(encoding="utf-8"))++ # Handle dict format (standard session)+ messages = []+ if isinstance(data, dict):+ # Check for 'messages' key (standard format)+ if "messages" in data:+ for msg in data["messages"]:+ role = msg.get("role", "unknown")+ content = msg.get("content", "")+ if content:+ messages.append(f"{role.upper()}: {content}")+ # Check for 'parts' key (checkpoint format sometimes)+ elif "parts" in data:+ messages.append(f"Saved Session Content: {data['parts']}")++ # Handle list format (some older array-based sessions)+ elif isinstance(data, list):+ for item in data:+ if isinstance(item, dict):+ role = item.get("role", "unknown")+ content = item.get("content", "") or item.get("parts", "")+ if content:+ messages.append(f"{role.upper()}: {content}")++ if not messages:+ return ""++ return f"File: {file_path.name}\n\n" + "\n\n".join(messages)++ def _parse_jsonl_session(self, file_path: Path) -> str:+ """Parse JSONL session file."""+ messages = []+ try:+ with open(file_path, encoding="utf-8") as f:+ for line in f:+ line = line.strip()+ if not line:+ continue+ try:+ data = json.loads(line)+ # Skip metadata lines if they don't have content+ if "role" in data and "content" in data:+ messages.append(f"{data['role'].upper()}: {data['content']}")+ elif "parts" in data: # sometimes parts is used+ messages.append(+ f"{data.get('role', 'unknown').upper()}: {data['parts']}"+ )+ except json.JSONDecodeError:+ continue+ except Exception:+ return ""++ if not messages:+ return ""++ return f"File: {file_path.name}\n\n" + "\n\n".join(messages)diff --git a/apps/gemini_rag.py b/apps/gemini_rag.pynew file mode 100644index 0000000..9578850--- /dev/null+++ b/apps/gemini_rag.py@@ -0,0 +1,69 @@+"""+Gemini CLI RAG example.+Indexes and searches Gemini CLI history (~/.gemini).+"""++import sys+from pathlib import Path+from typing import Any++# Add parent directory to path for imports+sys.path.insert(0, str(Path(__file__).parent))++from base_rag_example import BaseRAGExample+from chunking import create_text_chunks++from .gemini_data.gemini_reader import GeminiReader+++class GeminiRAG(BaseRAGExample):+ """RAG example for Gemini CLI history."""++ def __init__(self):+ super().__init__(+ name="Gemini CLI",+ description="Process and query Gemini CLI history with LEANN",+ default_index_name="gemini_index",+ )++ def _add_specific_arguments(self, parser):+ """Add Gemini-specific arguments."""+ group = parser.add_argument_group("Gemini Parameters")+ group.add_argument(+ "--gemini-path",+ type=str,+ default="~/.gemini",+ help="Path to .gemini directory (default: ~/.gemini)",+ )++ async def load_data(self, args) -> list[dict[str, Any]]:+ """Load Gemini history and convert to text chunks."""+ print(f"Loading Gemini history from: {args.gemini_path}")++ reader = GeminiReader()+ documents = reader.load_data(history_dir=args.gemini_path, max_count=args.max_items)++ if not documents:+ print("No documents found! Check if ~/.gemini exists and has history.")+ return []++ # Convert dicts to Document objects for chunking+ from llama_index.core import Document++ docs = [Document(text=d["text"], metadata=d["metadata"]) for d in documents]++ # Convert to text chunks+ print(f"splitting {len(documents)} documents into chunks...")+ chunks = create_text_chunks(docs)++ return chunks+++if __name__ == "__main__":+ import asyncio++ print("\n✨ Gemini CLI RAG")+ print("=" * 50)++ rag = GeminiRAG()+ asyncio.run(rag.run())diff --git a/apps/qwen_data/__init__.py b/apps/qwen_data/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/apps/qwen_data/qwen_reader.py b/apps/qwen_data/qwen_reader.pynew file mode 100644index 0000000..865121d--- /dev/null+++ b/apps/qwen_data/qwen_reader.py@@ -0,0 +1,150 @@+import json+import logging+from pathlib import Path+from typing import Any++logger = logging.getLogger(__name__)+++class QwenReader:+ """Reader for Qwen Code CLI history files."""++ def __init__(self):+ pass++ def load_data(self, history_dir: str, max_count: int = -1) -> list[dict[str, Any]]:+ """+ Load data from Qwen Code history directory.++ Args:+ history_dir: Path to .qwen-code directory+ max_count: Max number of conversations to load++ Returns:+ List of dictionaries with 'text' and 'metadata' keys+ """+ history_path = Path(history_dir).expanduser()+ if not history_path.exists():+ print(f"Qwen history directory not found: {history_path}")+ return []++ documents = []++ # 1. Load Memory (QWEN.md or MEMORY.md)+ for memory_filename in ["QWEN.md", "MEMORY.md"]:+ memory_file = history_path / memory_filename+ if memory_file.exists():+ try:+ text = memory_file.read_text(encoding="utf-8")+ if text.strip():+ documents.append(+ {+ "text": f"Qwen Code Memory:\n{text}",+ "metadata": {"source": str(memory_file), "type": "memory"},+ }+ )+ except Exception as e:+ print(f"Error reading memory file {memory_filename}: {e}")++ # 2. Find Session Files+ # Legacy JSON sessions+ session_files = list(history_path.glob("session-*.json"))+ # New JSONL sessions+ session_files.extend(list(history_path.glob("session-*.jsonl")))+ # Checkpoints+ session_files.extend(list(history_path.glob("checkpoint-*.json")))++ # Sort by modification time (newest first)+ session_files.sort(key=lambda x: x.stat().st_mtime, reverse=True)++ print(f"Found {len(session_files)} session files.")++ count = 0+ for file_path in session_files:+ if max_count > 0 and count >= max_count:+ break++ try:+ content = ""+ if file_path.suffix == ".jsonl":+ content = self._parse_jsonl_session(file_path)+ elif file_path.suffix == ".json":+ content = self._parse_json_session(file_path)++ if content:+ documents.append(+ {+ "text": content,+ "metadata": {+ "source": str(file_path),+ "type": "session",+ "filename": file_path.name,+ },+ }+ )+ count += 1+ except Exception as e:+ print(f"Error reading {file_path.name}: {e}")++ print(f"Successfully loaded {len(documents)} items from Qwen history.")+ return documents++ def _parse_json_session(self, file_path: Path) -> str:+ """Parse legacy JSON session file."""+ data = json.loads(file_path.read_text(encoding="utf-8"))++ # Handle dict format (standard session)+ messages = []+ if isinstance(data, dict):+ # Check for 'messages' key (standard format)+ if "messages" in data:+ for msg in data["messages"]:+ role = msg.get("role", "unknown")+ content = msg.get("content", "")+ if content:+ messages.append(f"{role.upper()}: {content}")+ # Check for 'parts' key (checkpoint format sometimes)+ elif "parts" in data:+ messages.append(f"Saved Session Content: {data['parts']}")++ # Handle list format (some older array-based sessions)+ elif isinstance(data, list):+ for item in data:+ if isinstance(item, dict):+ role = item.get("role", "unknown")+ content = item.get("content", "") or item.get("parts", "")+ if content:+ messages.append(f"{role.upper()}: {content}")++ if not messages:+ return ""++ return f"File: {file_path.name}\n\n" + "\n\n".join(messages)++ def _parse_jsonl_session(self, file_path: Path) -> str:+ """Parse JSONL session file."""+ messages = []+ try:+ with open(file_path, encoding="utf-8") as f:+ for line in f:+ line = line.strip()+ if not line:+ continue+ try:+ data = json.loads(line)+ # Skip metadata lines if they don't have content+ if "role" in data and "content" in data:+ messages.append(f"{data['role'].upper()}: {data['content']}")+ elif "parts" in data: # sometimes parts is used+ messages.append(+ f"{data.get('role', 'unknown').upper()}: {data['parts']}"+ )+ except json.JSONDecodeError:+ continue+ except Exception:+ return ""++ if not messages:+ return ""++ return f"File: {file_path.name}\n\n" + "\n\n".join(messages)diff --git a/apps/qwen_rag.py b/apps/qwen_rag.pynew file mode 100644index 0000000..ae665be--- /dev/null+++ b/apps/qwen_rag.py@@ -0,0 +1,69 @@+"""+Qwen Code RAG example.+Indexes and searches Qwen Code CLI history (~/.qwen-code).+"""++import sys+from pathlib import Path+from typing import Any++# Add parent directory to path for imports+sys.path.insert(0, str(Path(__file__).parent))++from base_rag_example import BaseRAGExample+from chunking import create_text_chunks++from .qwen_data.qwen_reader import QwenReader+++class QwenRAG(BaseRAGExample):+ """RAG example for Qwen Code CLI history."""++ def __init__(self):+ super().__init__(+ name="Qwen Code",+ description="Process and query Qwen Code CLI history with LEANN",+ default_index_name="qwen_index",+ )++ def _add_specific_arguments(self, parser):+ """Add Qwen-specific arguments."""+ group = parser.add_argument_group("Qwen Parameters")+ group.add_argument(+ "--qwen-path",+ type=str,+ default="~/.qwen-code",+ help="Path to .qwen-code directory (default: ~/.qwen-code)",+ )++ async def load_data(self, args) -> list[dict[str, Any]]:+ """Load Qwen history and convert to text chunks."""+ print(f"Loading Qwen history from: {args.qwen_path}")++ reader = QwenReader()+ documents = reader.load_data(history_dir=args.qwen_path, max_count=args.max_items)++ if not documents:+ print("No documents found! Check if ~/.qwen-code exists and has history.")+ return []++ # Convert dicts to Document objects for chunking+ from llama_index.core import Document++ docs = [Document(text=d["text"], metadata=d["metadata"]) for d in documents]++ # Convert to text chunks+ print(f"splitting {len(documents)} documents into chunks...")+ chunks = create_text_chunks(docs)++ return chunks+++if __name__ == "__main__":+ import asyncio++ print("\n✨ Qwen Code RAG")+ print("=" * 50)++ rag = QwenRAG()+ asyncio.run(rag.run())