Skip to content

A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.

Added Vision-Rag

Joel James520ca2c76db6
diff --git a/vision_rag/README.md b/vision_rag/README.md
new file mode 100644
index 0000000..e2ec88a
--- /dev/null
+++ b/vision_rag/README.md
@@ -0,0 +1,78 @@
+# VisionRAG: Multimodal Search & Visual Question Answering
+
+A powerful Streamlit application that combines multimodal search with visual question answering to analyze images and PDF documents using AI.
+
+## Features
+
+- **Multimodal Search**: Uses Cohere Embed-4 to find semantically relevant images for text questions
+- **Visual Question Answering**: Employs Google Gemini 2.5 Flash to analyze images and generate context-aware answers
+- **Flexible Content Sources**:
+ - Upload custom images (PNG, JPG, JPEG)
+ - Upload PDF documents (automatically extracts pages as images)
+ - Load sample financial charts and infographics
+- **No OCR Required**: Directly processes complex visual elements without text extraction
+
+## πŸš€ Quick Start
+
+### Prerequisites
+
+- Python 3.8+
+- Cohere API key
+- Google Gemini API key
+
+### Installation
+
+1. **Clone the repository**
+ ```bash
+ git clone <repository-url>
+ cd vision-rag
+ ```
+
+2. **Create and activate virtual environment**
+ ```bash
+ python -m venv venv
+ # On Windows
+ .\venv\Scripts\activate
+ # On macOS/Linux
+ source venv/bin/activate
+ ```
+
+3. **Install dependencies**
+ ```bash
+ pip install -r requirements.txt
+ ```
+
+4. **Set up API keys**
+ - Get your [Cohere API key](https://cohere.ai/)
+ - Get your [Google Gemini API key](https://makersuite.google.com/app/apikey)
+ - Create a `.env` file in the project root
+ - Add your API keys to the `.env` file:
+ ```
+ COHERE_API_KEY=your_cohere_api_key_here
+ GEMINI_API_KEY=your_gemini_api_key_here
+ ```
+
+5. **Run the application**
+ ```bash
+ streamlit run app.py
+ ```
+
+## Dependencies
+
+- **streamlit**: Web application framework
+- **cohere**: Embedding generation
+- **requests**: HTTP requests for Gemini API
+- **PyMuPDF**: PDF processing
+- **Pillow**: Image processing
+- **numpy**: Numerical operations
+- **scikit-learn**: Cosine similarity calculations
+
+
+## Contributing
+
+Contributions, issues, and feature requests are welcome. Please feel free to submit a Pull Request or open an issue.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
diff --git a/vision_rag/app.py b/vision_rag/app.py
new file mode 100644
index 0000000..36de46b
--- /dev/null
+++ b/vision_rag/app.py
@@ -0,0 +1,145 @@
+import streamlit as st
+import os
+import datetime
+from PIL import Image
+import numpy as np
+from dotenv import load_dotenv
+from utils import get_cohere_embedding, gemini_vqa, pdf_to_images, image_to_bytes, find_most_similar
+
+# Load environment variables
+load_dotenv()
+
+# Page configuration
+st.set_page_config(page_title="VisionRAG: Multimodal Search & VQA", layout="wide")
+st.title("VisionRAG: Multimodal Search & Visual Question Answering")
+
+# API Keys from environment variables
+cohere_api = os.getenv("COHERE_API_KEY")
+gemini_api = os.getenv("GEMINI_API_KEY")
+
+# Initialize session state
+if 'items' not in st.session_state:
+ st.session_state['items'] = [] # List of loaded images/PDF pages with embeddings
+
+if 'conversation' not in st.session_state:
+ st.session_state['conversation'] = [] # Chat history
+
+# Sidebar - Content Management
+st.sidebar.header("Content Sources")
+
+# File uploader
+uploaded_files = st.sidebar.file_uploader(
+ "Upload Images or PDFs",
+ type=["png", "jpg", "jpeg", "pdf"],
+ accept_multiple_files=True,
+ help="Upload images (PNG, JPG, JPEG) or PDF documents"
+)
+
+# Process uploaded files
+if uploaded_files:
+ new_items_count = 0
+ existing_names = set((item['name'], item['type']) for item in st.session_state['items'])
+
+ for file in uploaded_files:
+ if file.type.startswith('image/'):
+ key = (file.name, 'image')
+ if key not in existing_names:
+ img = Image.open(file).convert('RGB')
+ st.session_state['items'].append({'type': 'image', 'name': file.name, 'img': img, 'emb': None})
+ new_items_count += 1
+ elif file.type == 'application/pdf':
+ pdf_bytes = file.read()
+ pages = pdf_to_images(pdf_bytes)
+ for i, page_img in enumerate(pages):
+ name = f"{file.name} - Page {i+1}"
+ key = (name, 'pdf_page')
+ if key not in existing_names:
+ st.session_state['items'].append({'type': 'pdf_page', 'name': name, 'img': page_img, 'emb': None})
+ new_items_count += 1
+
+ st.success(f"Uploaded {len(uploaded_files)} file(s) with {new_items_count} total items.")
+
+# Sidebar - Content Preview
+st.sidebar.header("πŸ“ Loaded Content")
+if st.session_state['items']:
+ st.sidebar.markdown(f"**Total items:** {len(st.session_state['items'])}")
+ preview_items = st.session_state['items'][:6] # Show first 6 items
+
+ for item in preview_items:
+ with st.sidebar.container():
+ small_img = item['img'].resize((120, 120), Image.Resampling.LANCZOS)
+ caption = item['name'][:15] + "..." if len(item['name']) > 15 else item['name']
+ st.image(small_img, caption=caption, width=120)
+
+ if len(st.session_state['items']) > 6:
+ st.sidebar.markdown(f"*... and {len(st.session_state['items']) - 6} more items*")
+else:
+ st.sidebar.info("No content loaded yet.")
+
+# Main Interface - Chat
+st.subheader("Chat with Your Visual Data")
+
+# Display conversation history
+if st.session_state['conversation']:
+ st.markdown("### Conversation History")
+ for exchange in st.session_state['conversation']:
+ with st.expander(f"Q: {exchange['question']} ({exchange['timestamp']})", expanded=False):
+ st.markdown(f"**Question:** {exchange['question']}")
+ st.markdown(f"**Answer:** {exchange['answer']}")
+ st.markdown(f"**Relevant Image:** {exchange['relevant_image']}")
+ if exchange.get('image_display'):
+ st.image(exchange['image_display'], caption=exchange['relevant_image'], use_container_width=True)
+
+# Clear conversation button
+if st.session_state['conversation']:
+ if st.button("πŸ—‘οΈ Clear Conversation"):
+ st.session_state['conversation'] = []
+ st.rerun()
+
+# Question input form
+with st.form("question_form"):
+ question = st.text_input("Ask a question about your visual data:")
+ submit_button = st.form_submit_button("Send", type="primary")
+
+# Process question submission
+if submit_button:
+ if not cohere_api or not gemini_api:
+ st.error("Please provide both Cohere and Gemini API keys.")
+ elif not question:
+ st.error("Please enter a question.")
+ elif not st.session_state['items']:
+ st.error("No content loaded to search.")
+ else:
+ # Compute embeddings and find relevant image
+ with st.spinner("Computing embeddings and searching..."):
+ # Generate embeddings for all items if not already done
+ for item in st.session_state['items']:
+ if item['emb'] is None:
+ if item['type'] in ['image', 'pdf_page']:
+ img_bytes = image_to_bytes(item['img'])
+ item['emb'] = get_cohere_embedding(cohere_api, img_bytes, input_type='image')
+
+ # Get question embedding and find most similar image
+ q_emb = get_cohere_embedding(cohere_api, question, input_type='text')
+ emb_list = [item['emb'] for item in st.session_state['items']]
+ idx, sim = find_most_similar(q_emb, emb_list)
+ best_item = st.session_state['items'][idx]
+
+ # Generate answer using Gemini
+ st.info("Generating answer with Gemini...")
+ img_bytes = image_to_bytes(best_item['img'])
+ answer = gemini_vqa(gemini_api, img_bytes, question)
+
+ # Add to conversation history
+ timestamp = datetime.datetime.now().strftime("%H:%M:%S")
+ conversation_entry = {
+ 'question': question,
+ 'answer': answer,
+ 'relevant_image': best_item['name'],
+ 'timestamp': timestamp,
+ 'image_display': best_item['img'],
+ 'similarity': sim
+ }
+ st.session_state['conversation'].append(conversation_entry)
+
+ st.rerun()
\ No newline at end of file
diff --git a/vision_rag/requirements.txt b/vision_rag/requirements.txt
new file mode 100644
index 0000000..a031eb8
--- /dev/null
+++ b/vision_rag/requirements.txt
@@ -0,0 +1,8 @@
+streamlit
+cohere
+requests
+PyMuPDF
+Pillow
+numpy
+scikit-learn
+python-dotenv
\ No newline at end of file
diff --git a/vision_rag/utils.py b/vision_rag/utils.py
new file mode 100644
index 0000000..eba83c2
--- /dev/null
+++ b/vision_rag/utils.py
@@ -0,0 +1,134 @@
+"""
+Utility functions for VisionRAG: Multimodal Search & VQA application.
+Handles embeddings, VQA, PDF processing, and similarity calculations.
+"""
+
+import os
+import io
+import base64
+import numpy as np
+from PIL import Image
+import fitz # PyMuPDF
+import cohere
+import requests
+from sklearn.metrics.pairwise import cosine_similarity
+
+# --- Cohere Embed-4 ---
+def get_cohere_embedding(api_key, input_data, input_type='text'):
+ """
+ Get embedding for text or image using Cohere Embed-4.
+
+ Args:
+ api_key (str): Cohere API key
+ input_data (str or bytes): Text string or image bytes
+ input_type (str): 'text' or 'image'
+
+ Returns:
+ np.ndarray: Embedding vector
+ """
+ co = cohere.Client(api_key)
+
+ if input_type == 'text':
+ response = co.embed(texts=[input_data], model="embed-v4.0")
+ return np.array(response.embeddings[0])
+ elif input_type == 'image':
+ # Convert bytes to base64 data URI
+ image_b64 = base64.b64encode(input_data).decode('utf-8')
+ data_uri = f"data:image/png;base64,{image_b64}"
+ response = co.embed(images=[data_uri], model="embed-v4.0")
+ return np.array(response.embeddings[0])
+ else:
+ raise ValueError("input_type must be 'text' or 'image'")
+
+# --- Google Gemini 2.5 Flash VQA ---
+def gemini_vqa(api_key, image_bytes, question):
+ """
+ Send image and question to Gemini 2.5 Flash for Visual Question Answering.
+
+ Args:
+ api_key (str): Google Gemini API key
+ image_bytes (bytes): Image data
+ question (str): Question about the image
+
+ Returns:
+ str: Generated answer
+ """
+ endpoint = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key={api_key}"
+ headers = {"Content-Type": "application/json"}
+
+ image_b64 = base64.b64encode(image_bytes).decode()
+ data = {
+ "contents": [
+ {
+ "parts": [
+ {"inline_data": {"mime_type": "image/png", "data": image_b64}},
+ {"text": question}
+ ]
+ }
+ ]
+ }
+
+ response = requests.post(endpoint, headers=headers, json=data)
+
+ if response.status_code == 200:
+ return response.json()['candidates'][0]['content']['parts'][0]['text']
+ else:
+ return f"Error: {response.status_code} {response.text}"
+
+# --- PDF to Images ---
+def pdf_to_images(pdf_bytes):
+ """
+ Convert PDF bytes to a list of PIL Images (one per page).
+
+ Args:
+ pdf_bytes (bytes): PDF file data
+
+ Returns:
+ list: List of PIL Image objects
+ """
+ doc = fitz.open(stream=pdf_bytes, filetype="pdf")
+ images = []
+
+ for page_num in range(len(doc)):
+ page = doc.load_page(page_num)
+ pix = page.get_pixmap(matrix=fitz.Matrix(2, 2)) # 2x zoom for better quality
+ img_data = pix.tobytes("png")
+ img = Image.open(io.BytesIO(img_data))
+ images.append(img)
+
+ doc.close()
+ return images
+
+# --- Image to Bytes ---
+def image_to_bytes(img, format='PNG'):
+ """
+ Convert PIL Image to bytes.
+
+ Args:
+ img (PIL.Image): Image to convert
+ format (str): Image format (default: 'PNG')
+
+ Returns:
+ bytes: Image data
+ """
+ buf = io.BytesIO()
+ img.save(buf, format=format)
+ return buf.getvalue()
+
+# --- Cosine Similarity ---
+def find_most_similar(query_emb, emb_list):
+ """
+ Find the most similar embedding in a list to the query embedding.
+
+ Args:
+ query_emb (np.ndarray): Query embedding vector
+ emb_list (list): List of embedding vectors to compare against
+
+ Returns:
+ tuple: (index of most similar embedding, similarity score)
+ """
+ similarities = cosine_similarity([query_emb], emb_list)[0]
+ best_idx = int(np.argmax(similarities))
+ best_score = float(np.max(similarities))
+
+ return best_idx, best_score
\ No newline at end of file