A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.
fix(graphrag_knowledge_system): remove unresolved merge conflict markers
Rudy Celeklib730ab79595d
diff --git a/rag_apps/graphrag_knowledge_system/.env.example b/rag_apps/graphrag_knowledge_system/.env.exampleindex 570cab1..54aa524 100644--- a/rag_apps/graphrag_knowledge_system/.env.example+++ b/rag_apps/graphrag_knowledge_system/.env.example@@ -1,12 +1,6 @@-<<<<<<< HEAD-MISTRAL_API_KEY=your_mistral_api_key_here--# Optional — defaults to http://localhost:11434-=======# Mistral API key used for entity extraction and answer generation.# Get one free at https://platform.mistral.aiMISTRAL_API_KEY=your_mistral_api_key_here# Optional, defaults to http://localhost:11434->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4a# OLLAMA_BASE_URL=http://localhost:11434diff --git a/rag_apps/graphrag_knowledge_system/app.py b/rag_apps/graphrag_knowledge_system/app.pyindex 3161982..12c8aba 100644--- a/rag_apps/graphrag_knowledge_system/app.py+++ b/rag_apps/graphrag_knowledge_system/app.py@@ -1,11 +1,8 @@-<<<<<<< HEAD-======="""GraphRAG Knowledge System Streamlit app: document ingestion, knowledge graphconstruction, community detection, and Local/Global search over the graph."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aimport uuidimport jsonimport os@@ -77,10 +74,7 @@ st.markdown(@st.cache_resource(show_spinner=False)def get_components():-<<<<<<< HEAD-======="""Construct and cache the pipeline components shared across reruns."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4agraph_store = GraphStore()vector_store = VectorStore()entity_extractor = EntityExtractor()@@ -117,10 +111,7 @@ Summary:"""def run_community_detection(graph_store: GraphStore, llm_client: LLMClient, status_fn) -> int:-<<<<<<< HEAD-======="""Detect entity communities in the graph and summarise each one with the LLM."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4afull_graph = graph_store.get_full_graph()entities = full_graph["entities"]relationships = full_graph["relationships"]@@ -195,20 +186,13 @@ def ingest_document(cache_manager: CacheManager,llm_client: LLMClient,) -> dict:-<<<<<<< HEAD-======="""Chunk, extract entities from, and index a single uploaded document."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4afile_hash = cache_manager.file_hash(file_content)if cache_manager.is_processed(file_hash):return {"status": "cached",-<<<<<<< HEAD- "message": f"'{filename}' was already processed — skipping.",-======="message": f"'{filename}' was already processed, skipping.",->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4a}# ── Step 1: Load and chunk ────────────────────────────────────────────────@@ -238,11 +222,7 @@ def ingest_document(chunk_id = f"{doc_id}_chunk_{idx}"chunk_text = chunk.page_content-<<<<<<< HEAD- # Store chunk in Neo4j-=======# Store chunk in the knowledge graph->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4agraph_store.create_chunk(chunk_id, chunk_text, doc_id)# Extract entities and relationships@@ -324,10 +304,7 @@ TYPE_COLORS = {def _type_badge(entity_type: str) -> str:-<<<<<<< HEAD-======="""Render an entity type as a small colored HTML badge."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4acolor = TYPE_COLORS.get(entity_type, "#888")return (f'<span style="background:{color};color:#fff;padding:2px 10px;'@@ -337,10 +314,7 @@ def _type_badge(entity_type: str) -> str:def render_entity_list(entities: list):-<<<<<<< HEAD-======="""Render up to 20 entities as name, description, and type badge rows."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aif not entities:st.caption("No entities retrieved.")return@@ -359,10 +333,7 @@ def render_entity_list(entities: list):def render_relationship_list(relationships: list):-<<<<<<< HEAD-======="""Render up to 15 relationships as source-target-weight rows."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aif not relationships:st.caption("No relationships retrieved.")return@@ -381,10 +352,7 @@ def render_relationship_list(relationships: list):def main():-<<<<<<< HEAD-======="""Render the Streamlit page: ingestion sidebar, search controls, and results."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4a# ── Header ────────────────────────────────────────────────────────────────st.markdown('<div class="main-header">🕸️ GraphRAG Knowledge System</div>', unsafe_allow_html=True@@ -408,13 +376,8 @@ def main():except Exception as e:st.error(f"Failed to initialise components: {e}")st.info(-<<<<<<< HEAD- "Check your `.env` file — make sure NEO4J_URI, NEO4J_PASSWORD, "- "MISTRAL_API_KEY are set and Ollama is running on http://localhost:11434"-======="Check your `.env` file, make sure MISTRAL_API_KEY is set ""and Ollama is running on http://localhost:11434"->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4a)get_components.clear()return@@ -440,11 +403,7 @@ def main():"Upload PDF or TXT files",type=["pdf", "txt"],accept_multiple_files=True,-<<<<<<< HEAD- help="Documents are chunked, entity-extracted, and stored in Neo4j + ChromaDB.",-=======help="Documents are chunked, entity-extracted, and stored in the local knowledge graph and ChromaDB.",->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4a)if uploaded_files and st.button("⚙️ Process Documents", type="primary", use_container_width=True):@@ -486,21 +445,13 @@ def main():try:st.metric("Chunks", vector_store.count())except Exception:-<<<<<<< HEAD- st.metric("Chunks", "—")-=======st.metric("Chunks", "N/A")->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4awith col2:try:communities = graph_store.get_all_communities()st.metric("Communities", len(communities))except Exception:-<<<<<<< HEAD- st.metric("Communities", "—")-=======st.metric("Communities", "N/A")->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aif st.button("🔄 Refresh Stats", use_container_width=True):st.rerun()@@ -546,11 +497,7 @@ def main():# ── Answer ────────────────────────────────────────────────────────────mode_label = "Local Search" if "Local" in search_mode else "Global Search"-<<<<<<< HEAD- st.markdown(f"### Answer — {mode_label}")-=======st.markdown(f"### Answer: {mode_label}")->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4ast.markdown(result["answer"])# ── Supporting context ────────────────────────────────────────────────@@ -597,11 +544,7 @@ def main():with info_col2:st.info("**2. Knowledge Graph Built**\n\n"-<<<<<<< HEAD- "Entities and relationships are stored in Neo4j. "-======="Entities and relationships are stored in the local knowledge graph. "->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4a"Text embeddings go into ChromaDB via Ollama.")with info_col3:diff --git a/rag_apps/graphrag_knowledge_system/src/cache_manager.py b/rag_apps/graphrag_knowledge_system/src/cache_manager.pyindex f775fce..289b4c0 100644--- a/rag_apps/graphrag_knowledge_system/src/cache_manager.py+++ b/rag_apps/graphrag_knowledge_system/src/cache_manager.py@@ -6,18 +6,12 @@ from pathlib import Pathclass CacheManager:def __init__(self, cache_path: str = "./cache.json"):-<<<<<<< HEAD-======="""Load the processed-file cache from disk."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aself.cache_path = Path(cache_path)self._load()def _load(self):-<<<<<<< HEAD-======="""Read the cache JSON file from disk, or start empty if none exists."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aif self.cache_path.exists():with open(self.cache_path, "r") as f:self.cache = json.load(f)@@ -25,34 +19,22 @@ class CacheManager:self.cache = {}def _save(self):-<<<<<<< HEAD-======="""Write the current cache to disk as JSON."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4awith open(self.cache_path, "w") as f:json.dump(self.cache, f, indent=2)def file_hash(self, content: bytes) -> str:-<<<<<<< HEAD- return hashlib.sha256(content).hexdigest()-- def is_processed(self, file_hash: str) -> bool:-======="""Compute a SHA-256 hash of the given file content."""return hashlib.sha256(content).hexdigest()def is_processed(self, file_hash: str) -> bool:"""Check whether a file with this hash has already been indexed."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4areturn file_hash in self.cachedef mark_processed(self, file_hash: str, filename: str, doc_id: str, chunk_count: int):-<<<<<<< HEAD-======="""Record a file as processed and persist the cache."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aself.cache[file_hash] = {"filename": filename,"document_id": doc_id,@@ -62,10 +44,7 @@ class CacheManager:self._save()def get_all(self) -> list:-<<<<<<< HEAD-======="""Return summary info for every processed file in the cache."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4areturn [{"filename": v["filename"],@@ -77,10 +56,7 @@ class CacheManager:]def remove(self, file_hash: str):-<<<<<<< HEAD-======="""Remove a file's cache entry, if present, and persist the change."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aif file_hash in self.cache:del self.cache[file_hash]self._save()diff --git a/rag_apps/graphrag_knowledge_system/src/document_processor.py b/rag_apps/graphrag_knowledge_system/src/document_processor.pyindex c40e429..c34b169 100644--- a/rag_apps/graphrag_knowledge_system/src/document_processor.py+++ b/rag_apps/graphrag_knowledge_system/src/document_processor.py@@ -8,10 +8,7 @@ from langchain_community.document_loaders import PyPDFLoader, TextLoaderclass DocumentProcessor:def __init__(self, chunk_size: int = 800, chunk_overlap: int = 150):-<<<<<<< HEAD-======="""Configure the text splitter used to chunk loaded documents."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aself.splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size,chunk_overlap=chunk_overlap,@@ -19,10 +16,7 @@ class DocumentProcessor:)def process(self, file_content: bytes, filename: str) -> list:-<<<<<<< HEAD-======="""Load a PDF or TXT file from bytes and split it into text chunks."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4asuffix = Path(filename).suffix.lower()if suffix not in {".pdf", ".txt"}:raise ValueError(f"Unsupported file type: {suffix}. Use PDF or TXT.")diff --git a/rag_apps/graphrag_knowledge_system/src/entity_extractor.py b/rag_apps/graphrag_knowledge_system/src/entity_extractor.pyindex 0cdebbd..615876c 100644--- a/rag_apps/graphrag_knowledge_system/src/entity_extractor.py+++ b/rag_apps/graphrag_knowledge_system/src/entity_extractor.py@@ -28,11 +28,7 @@ Extraction rules:- Extract 3–12 meaningful, specific entities; skip generic words- Every relationship source and target MUST be an entity you extracted- weight is 0.0–1.0 reflecting relationship strength or importance-<<<<<<< HEAD-- Return only valid JSON — no markdown fences, no extra text-=======- Return only valid JSON, no markdown fences, no extra text->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aText to analyze:{text}"""@@ -40,17 +36,11 @@ Text to analyze:class EntityExtractor:def __init__(self):-<<<<<<< HEAD- self.llm = LLMClient()-- def extract(self, text: str, chunk_id: str) -> dict:-======="""Create the LLM client used for entity and relationship extraction."""self.llm = LLMClient()def extract(self, text: str, chunk_id: str) -> dict:"""Extract typed entities and weighted relationships from a text chunk via Mistral."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aprompt = EXTRACTION_PROMPT.format(text=text[:3000])try:diff --git a/rag_apps/graphrag_knowledge_system/src/graph_store.py b/rag_apps/graphrag_knowledge_system/src/graph_store.pyindex e1210c8..3012fc7 100644--- a/rag_apps/graphrag_knowledge_system/src/graph_store.py+++ b/rag_apps/graphrag_knowledge_system/src/graph_store.py@@ -12,20 +12,14 @@ class GraphStore:"""def __init__(self, persist_path: str = "./graph_store.json"):-<<<<<<< HEAD-======="""Load the persisted graph from disk, or start a fresh one if none exists."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aself.persist_path = Path(persist_path)self.G: nx.DiGraph = self._load()# ── Persistence ───────────────────────────────────────────────────────────def _load(self) -> nx.DiGraph:-<<<<<<< HEAD-======="""Read the graph JSON file from disk and rebuild it as a DiGraph."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aif self.persist_path.exists():try:with open(self.persist_path, "r", encoding="utf-8") as f:@@ -36,10 +30,7 @@ class GraphStore:return nx.DiGraph()def _save(self):-<<<<<<< HEAD-======="""Serialise the current graph to disk as JSON."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4adata = nx.node_link_data(self.G)with open(self.persist_path, "w", encoding="utf-8") as f:json.dump(data, f, ensure_ascii=False)@@ -47,20 +38,14 @@ class GraphStore:# ── Document ─────────────────────────────────────────────────────────────def create_document(self, doc_id: str, filename: str, file_hash: str):-<<<<<<< HEAD-======="""Add a document node to the graph and persist the change."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aself.G.add_node(doc_id, node_type="document", filename=filename, file_hash=file_hash)self._save()def document_exists(self, file_hash: str) -> bool:-<<<<<<< HEAD-======="""Check whether a document with the given file hash is already in the graph."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4areturn any(d.get("node_type") == "document" and d.get("file_hash") == file_hashfor _, d in self.G.nodes(data=True)@@ -69,10 +54,7 @@ class GraphStore:# ── Chunks ────────────────────────────────────────────────────────────────def create_chunk(self, chunk_id: str, content: str, doc_id: str):-<<<<<<< HEAD-======="""Add a chunk node linked to its parent document and persist the change."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aself.G.add_node(chunk_id, node_type="chunk", content=content, doc_id=doc_id)if self.G.has_node(doc_id):self.G.add_edge(chunk_id, doc_id, rel_type="PART_OF")@@ -83,10 +65,7 @@ class GraphStore:def upsert_entity(self, name: str, entity_type: str, description: str, chunk_id: str):-<<<<<<< HEAD-======="""Create or update an entity node and link it to the chunk it appeared in."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aif self.G.has_node(name) and self.G.nodes[name].get("node_type") == "entity":existing = self.G.nodes[name]if len(description) > len(existing.get("description", "")):@@ -109,10 +88,7 @@ class GraphStore:def upsert_relationship(self, source: str, target: str, description: str, weight: float):-<<<<<<< HEAD-======="""Create a relationship edge between two entities, or average the weight if it exists."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aif not self.G.has_node(source) or not self.G.has_node(target):returnif self.G.has_edge(source, target):@@ -134,22 +110,15 @@ class GraphStore:# ── Community ─────────────────────────────────────────────────────────────def set_entity_community(self, entity_name: str, community_id: str):-<<<<<<< HEAD- # No _save() here — called in a tight loop; caller saves via create_community()-======="""Tag an entity node with the community it belongs to."""# No _save() here since this runs in a tight loop; caller saves via create_community()->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aif self.G.has_node(entity_name):self.G.nodes[entity_name]["community_id"] = community_iddef create_community(self, community_id: str, entity_names: List[str], summary: str, level: int = 0):-<<<<<<< HEAD-======="""Add a community node, link its member entities, and persist the change."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aself.G.add_node(community_id,node_type="community",@@ -163,10 +132,7 @@ class GraphStore:self._save()def get_all_communities(self) -> List[Dict]:-<<<<<<< HEAD-======="""Return every community node along with its member entity names."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aresult = []for node_id, data in self.G.nodes(data=True):if data.get("node_type") != "community":@@ -188,10 +154,7 @@ class GraphStore:return resultdef communities_exist(self) -> bool:-<<<<<<< HEAD-======="""Check whether any community nodes have been created yet."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4areturn any(d.get("node_type") == "community" for _, d in self.G.nodes(data=True))@@ -199,10 +162,7 @@ class GraphStore:# ── Graph export for community detection ─────────────────────────────────def get_full_graph(self) -> Dict:-<<<<<<< HEAD-======="""Return every entity and weighted relationship in the graph for community detection."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aentities = [{"name": n, "type": d.get("type", "")}for n, d in self.G.nodes(data=True)@@ -216,10 +176,7 @@ class GraphStore:return {"entities": entities, "relationships": relationships}def get_entity_descriptions(self, names: List[str]) -> List[Dict]:-<<<<<<< HEAD-======="""Look up name, type, and description for each entity in the given list."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aresult = []for name in names:if not self.G.has_node(name):@@ -238,10 +195,7 @@ class GraphStore:# ── Search context ────────────────────────────────────────────────────────def get_entity_neighborhood(self, entity_names: List[str]) -> Dict:-<<<<<<< HEAD-======="""Collect the given entities plus everything directly related to them."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aseen_entities: set = set()seen_rels: set = set()entities: List[Dict] = []@@ -293,15 +247,9 @@ class GraphStore:# ── Helpers ───────────────────────────────────────────────────────────────def ping(self) -> Tuple[bool, str]:-<<<<<<< HEAD- return True, "Connected (local NetworkX graph)"-- def close(self):-======="""Report that the local graph store is always available."""return True, "Connected (local NetworkX graph)"def close(self):"""Persist the graph to disk before shutting down."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aself._save()diff --git a/rag_apps/graphrag_knowledge_system/src/llm_client.py b/rag_apps/graphrag_knowledge_system/src/llm_client.pyindex 452bcc8..3c570ae 100644--- a/rag_apps/graphrag_knowledge_system/src/llm_client.py+++ b/rag_apps/graphrag_knowledge_system/src/llm_client.py@@ -6,10 +6,7 @@ from langchain_core.messages import HumanMessage, SystemMessageclass LLMClient:def __init__(self):-<<<<<<< HEAD-======="""Create the plain-text and JSON-mode Mistral chat clients."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aapi_key = os.getenv("MISTRAL_API_KEY")if not api_key:raise ValueError("MISTRAL_API_KEY not set in environment")@@ -26,10 +23,7 @@ class LLMClient:)def complete(self, prompt: str, json_mode: bool = False, max_retries: int = 3) -> str:-<<<<<<< HEAD-======="""Send a single prompt to Mistral, retrying with backoff on failure."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aclient = self.llm_json if json_mode else self.llmmessages = [HumanMessage(content=prompt)]for attempt in range(max_retries):@@ -45,10 +39,7 @@ class LLMClient:def complete_with_system(self, system: str, user: str, json_mode: bool = False) -> str:-<<<<<<< HEAD-======="""Send a system and user message pair to Mistral and return the response text."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aclient = self.llm_json if json_mode else self.llmmessages = [SystemMessage(content=system), HumanMessage(content=user)]response = client.invoke(messages)diff --git a/rag_apps/graphrag_knowledge_system/src/search_engine.py b/rag_apps/graphrag_knowledge_system/src/search_engine.pyindex bb082db..10cf18d 100644--- a/rag_apps/graphrag_knowledge_system/src/search_engine.py+++ b/rag_apps/graphrag_knowledge_system/src/search_engine.py@@ -59,10 +59,7 @@ Answer:"""class SearchEngine:def __init__(self, graph_store: GraphStore, vector_store: VectorStore):-<<<<<<< HEAD-======="""Wire up the graph store, vector store, and LLM client used for search."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aself.graph = graph_storeself.vector = vector_storeself.llm = LLMClient()@@ -70,10 +67,7 @@ class SearchEngine:# ── Local Search ──────────────────────────────────────────────────────────def local_search(self, query: str, n_chunks: int = 5) -> Dict:-<<<<<<< HEAD-======="""Retrieve matching chunks and their graph context, then answer the question."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4aif self.vector.count() == 0:return self._no_docs_result()@@ -132,10 +126,7 @@ class SearchEngine:# ── Global Search ─────────────────────────────────────────────────────────def global_search(self, query: str) -> Dict:-<<<<<<< HEAD-======="""Rate each community's relevance to the question and synthesise an answer from the top ones."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4acommunities = self.graph.get_all_communities()if not communities:return self.local_search(query)@@ -208,10 +199,7 @@ class SearchEngine:@staticmethoddef _no_docs_result() -> Dict:-<<<<<<< HEAD-======="""Build the placeholder result returned when no documents have been indexed."""->>>>>>> 1d1e9f137cfd1123edbae5d8e955ce0b9c7fcf4areturn {"answer": "No documents have been indexed yet. Please upload and process documents first.","entities": [],