A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.
Added Vision-Rag
Joel James520ca2c76db6
diff --git a/vision_rag/README.md b/vision_rag/README.mdnew file mode 100644index 0000000..e2ec88a--- /dev/null+++ b/vision_rag/README.md@@ -0,0 +1,78 @@+# VisionRAG: Multimodal Search & Visual Question Answering++A powerful Streamlit application that combines multimodal search with visual question answering to analyze images and PDF documents using AI.++## Features++- **Multimodal Search**: Uses Cohere Embed-4 to find semantically relevant images for text questions+- **Visual Question Answering**: Employs Google Gemini 2.5 Flash to analyze images and generate context-aware answers+- **Flexible Content Sources**:+ - Upload custom images (PNG, JPG, JPEG)+ - Upload PDF documents (automatically extracts pages as images)+ - Load sample financial charts and infographics+- **No OCR Required**: Directly processes complex visual elements without text extraction++## π Quick Start++### Prerequisites++- Python 3.8++- Cohere API key+- Google Gemini API key++### Installation++1. **Clone the repository**+ ```bash+ git clone <repository-url>+ cd vision-rag+ ```++2. **Create and activate virtual environment**+ ```bash+ python -m venv venv+ # On Windows+ .\venv\Scripts\activate+ # On macOS/Linux+ source venv/bin/activate+ ```++3. **Install dependencies**+ ```bash+ pip install -r requirements.txt+ ```++4. **Set up API keys**+ - Get your [Cohere API key](https://cohere.ai/)+ - Get your [Google Gemini API key](https://makersuite.google.com/app/apikey)+ - Create a `.env` file in the project root+ - Add your API keys to the `.env` file:+ ```+ COHERE_API_KEY=your_cohere_api_key_here+ GEMINI_API_KEY=your_gemini_api_key_here+ ```++5. **Run the application**+ ```bash+ streamlit run app.py+ ```++## Dependencies++- **streamlit**: Web application framework+- **cohere**: Embedding generation+- **requests**: HTTP requests for Gemini API+- **PyMuPDF**: PDF processing+- **Pillow**: Image processing+- **numpy**: Numerical operations+- **scikit-learn**: Cosine similarity calculations+++## Contributing++Contributions, issues, and feature requests are welcome. Please feel free to submit a Pull Request or open an issue.++## License++This project is licensed under the MIT License - see the LICENSE file for details.+diff --git a/vision_rag/app.py b/vision_rag/app.pynew file mode 100644index 0000000..36de46b--- /dev/null+++ b/vision_rag/app.py@@ -0,0 +1,145 @@+import streamlit as st+import os+import datetime+from PIL import Image+import numpy as np+from dotenv import load_dotenv+from utils import get_cohere_embedding, gemini_vqa, pdf_to_images, image_to_bytes, find_most_similar++# Load environment variables+load_dotenv()++# Page configuration+st.set_page_config(page_title="VisionRAG: Multimodal Search & VQA", layout="wide")+st.title("VisionRAG: Multimodal Search & Visual Question Answering")++# API Keys from environment variables+cohere_api = os.getenv("COHERE_API_KEY")+gemini_api = os.getenv("GEMINI_API_KEY")++# Initialize session state+if 'items' not in st.session_state:+ st.session_state['items'] = [] # List of loaded images/PDF pages with embeddings++if 'conversation' not in st.session_state:+ st.session_state['conversation'] = [] # Chat history++# Sidebar - Content Management+st.sidebar.header("Content Sources")++# File uploader+uploaded_files = st.sidebar.file_uploader(+ "Upload Images or PDFs",+ type=["png", "jpg", "jpeg", "pdf"],+ accept_multiple_files=True,+ help="Upload images (PNG, JPG, JPEG) or PDF documents"+)++# Process uploaded files+if uploaded_files:+ new_items_count = 0+ existing_names = set((item['name'], item['type']) for item in st.session_state['items'])++ for file in uploaded_files:+ if file.type.startswith('image/'):+ key = (file.name, 'image')+ if key not in existing_names:+ img = Image.open(file).convert('RGB')+ st.session_state['items'].append({'type': 'image', 'name': file.name, 'img': img, 'emb': None})+ new_items_count += 1+ elif file.type == 'application/pdf':+ pdf_bytes = file.read()+ pages = pdf_to_images(pdf_bytes)+ for i, page_img in enumerate(pages):+ name = f"{file.name} - Page {i+1}"+ key = (name, 'pdf_page')+ if key not in existing_names:+ st.session_state['items'].append({'type': 'pdf_page', 'name': name, 'img': page_img, 'emb': None})+ new_items_count += 1++ st.success(f"Uploaded {len(uploaded_files)} file(s) with {new_items_count} total items.")++# Sidebar - Content Preview+st.sidebar.header("π Loaded Content")+if st.session_state['items']:+ st.sidebar.markdown(f"**Total items:** {len(st.session_state['items'])}")+ preview_items = st.session_state['items'][:6] # Show first 6 items++ for item in preview_items:+ with st.sidebar.container():+ small_img = item['img'].resize((120, 120), Image.Resampling.LANCZOS)+ caption = item['name'][:15] + "..." if len(item['name']) > 15 else item['name']+ st.image(small_img, caption=caption, width=120)++ if len(st.session_state['items']) > 6:+ st.sidebar.markdown(f"*... and {len(st.session_state['items']) - 6} more items*")+else:+ st.sidebar.info("No content loaded yet.")++# Main Interface - Chat+st.subheader("Chat with Your Visual Data")++# Display conversation history+if st.session_state['conversation']:+ st.markdown("### Conversation History")+ for exchange in st.session_state['conversation']:+ with st.expander(f"Q: {exchange['question']} ({exchange['timestamp']})", expanded=False):+ st.markdown(f"**Question:** {exchange['question']}")+ st.markdown(f"**Answer:** {exchange['answer']}")+ st.markdown(f"**Relevant Image:** {exchange['relevant_image']}")+ if exchange.get('image_display'):+ st.image(exchange['image_display'], caption=exchange['relevant_image'], use_container_width=True)++# Clear conversation button+if st.session_state['conversation']:+ if st.button("ποΈ Clear Conversation"):+ st.session_state['conversation'] = []+ st.rerun()++# Question input form+with st.form("question_form"):+ question = st.text_input("Ask a question about your visual data:")+ submit_button = st.form_submit_button("Send", type="primary")++# Process question submission+if submit_button:+ if not cohere_api or not gemini_api:+ st.error("Please provide both Cohere and Gemini API keys.")+ elif not question:+ st.error("Please enter a question.")+ elif not st.session_state['items']:+ st.error("No content loaded to search.")+ else:+ # Compute embeddings and find relevant image+ with st.spinner("Computing embeddings and searching..."):+ # Generate embeddings for all items if not already done+ for item in st.session_state['items']:+ if item['emb'] is None:+ if item['type'] in ['image', 'pdf_page']:+ img_bytes = image_to_bytes(item['img'])+ item['emb'] = get_cohere_embedding(cohere_api, img_bytes, input_type='image')++ # Get question embedding and find most similar image+ q_emb = get_cohere_embedding(cohere_api, question, input_type='text')+ emb_list = [item['emb'] for item in st.session_state['items']]+ idx, sim = find_most_similar(q_emb, emb_list)+ best_item = st.session_state['items'][idx]++ # Generate answer using Gemini+ st.info("Generating answer with Gemini...")+ img_bytes = image_to_bytes(best_item['img'])+ answer = gemini_vqa(gemini_api, img_bytes, question)++ # Add to conversation history+ timestamp = datetime.datetime.now().strftime("%H:%M:%S")+ conversation_entry = {+ 'question': question,+ 'answer': answer,+ 'relevant_image': best_item['name'],+ 'timestamp': timestamp,+ 'image_display': best_item['img'],+ 'similarity': sim+ }+ st.session_state['conversation'].append(conversation_entry)++ st.rerun()\ No newline at end of filediff --git a/vision_rag/requirements.txt b/vision_rag/requirements.txtnew file mode 100644index 0000000..a031eb8--- /dev/null+++ b/vision_rag/requirements.txt@@ -0,0 +1,8 @@+streamlit+cohere+requests+PyMuPDF+Pillow+numpy+scikit-learn+python-dotenv\ No newline at end of filediff --git a/vision_rag/utils.py b/vision_rag/utils.pynew file mode 100644index 0000000..eba83c2--- /dev/null+++ b/vision_rag/utils.py@@ -0,0 +1,134 @@+"""+Utility functions for VisionRAG: Multimodal Search & VQA application.+Handles embeddings, VQA, PDF processing, and similarity calculations.+"""++import os+import io+import base64+import numpy as np+from PIL import Image+import fitz # PyMuPDF+import cohere+import requests+from sklearn.metrics.pairwise import cosine_similarity++# --- Cohere Embed-4 ---+def get_cohere_embedding(api_key, input_data, input_type='text'):+ """+ Get embedding for text or image using Cohere Embed-4.++ Args:+ api_key (str): Cohere API key+ input_data (str or bytes): Text string or image bytes+ input_type (str): 'text' or 'image'++ Returns:+ np.ndarray: Embedding vector+ """+ co = cohere.Client(api_key)++ if input_type == 'text':+ response = co.embed(texts=[input_data], model="embed-v4.0")+ return np.array(response.embeddings[0])+ elif input_type == 'image':+ # Convert bytes to base64 data URI+ image_b64 = base64.b64encode(input_data).decode('utf-8')+ data_uri = f"data:image/png;base64,{image_b64}"+ response = co.embed(images=[data_uri], model="embed-v4.0")+ return np.array(response.embeddings[0])+ else:+ raise ValueError("input_type must be 'text' or 'image'")++# --- Google Gemini 2.5 Flash VQA ---+def gemini_vqa(api_key, image_bytes, question):+ """+ Send image and question to Gemini 2.5 Flash for Visual Question Answering.++ Args:+ api_key (str): Google Gemini API key+ image_bytes (bytes): Image data+ question (str): Question about the image++ Returns:+ str: Generated answer+ """+ endpoint = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key={api_key}"+ headers = {"Content-Type": "application/json"}++ image_b64 = base64.b64encode(image_bytes).decode()+ data = {+ "contents": [+ {+ "parts": [+ {"inline_data": {"mime_type": "image/png", "data": image_b64}},+ {"text": question}+ ]+ }+ ]+ }++ response = requests.post(endpoint, headers=headers, json=data)++ if response.status_code == 200:+ return response.json()['candidates'][0]['content']['parts'][0]['text']+ else:+ return f"Error: {response.status_code} {response.text}"++# --- PDF to Images ---+def pdf_to_images(pdf_bytes):+ """+ Convert PDF bytes to a list of PIL Images (one per page).++ Args:+ pdf_bytes (bytes): PDF file data++ Returns:+ list: List of PIL Image objects+ """+ doc = fitz.open(stream=pdf_bytes, filetype="pdf")+ images = []++ for page_num in range(len(doc)):+ page = doc.load_page(page_num)+ pix = page.get_pixmap(matrix=fitz.Matrix(2, 2)) # 2x zoom for better quality+ img_data = pix.tobytes("png")+ img = Image.open(io.BytesIO(img_data))+ images.append(img)++ doc.close()+ return images++# --- Image to Bytes ---+def image_to_bytes(img, format='PNG'):+ """+ Convert PIL Image to bytes.++ Args:+ img (PIL.Image): Image to convert+ format (str): Image format (default: 'PNG')++ Returns:+ bytes: Image data+ """+ buf = io.BytesIO()+ img.save(buf, format=format)+ return buf.getvalue()++# --- Cosine Similarity ---+def find_most_similar(query_emb, emb_list):+ """+ Find the most similar embedding in a list to the query embedding.++ Args:+ query_emb (np.ndarray): Query embedding vector+ emb_list (list): List of embedding vectors to compare against++ Returns:+ tuple: (index of most similar embedding, similarity score)+ """+ similarities = cosine_similarity([query_emb], emb_list)[0]+ best_idx = int(np.argmax(similarities))+ best_score = float(np.max(similarities))++ return best_idx, best_score\ No newline at end of file