A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.
"""
VisionRAG: Multimodal Search & Visual Question Answering Streamlit app.
"""
import streamlit as st
import os
import datetime
from PIL import Image
import numpy as np
from dotenv import load_dotenv, find_dotenv
from utils import get_cohere_embedding, mistral_vqa, pdf_to_images, image_to_bytes, find_most_similar
# Load environment variables
load_dotenv(find_dotenv())
# Page configuration
st.set_page_config(page_title="VisionRAG: Multimodal Search & VQA", layout="wide")
st.title("VisionRAG: Multimodal Search & Visual Question Answering")
cohere_api = os.getenv("COHERE_API_KEY")
mistral_api = os.getenv("MISTRAL_API_KEY")
# Initialize session state
if 'items' not in st.session_state:
st.session_state['items'] = [] # List of loaded images/PDF pages with embeddings
if 'conversation' not in st.session_state:
st.session_state['conversation'] = [] # Chat history
# Sidebar - Content Management
st.sidebar.header("Content Sources")
# File uploader
uploaded_files = st.sidebar.file_uploader(
"Upload Images or PDFs",
type=["png", "jpg", "jpeg", "pdf"],
accept_multiple_files=True,
help="Upload images (PNG, JPG, JPEG) or PDF documents"
)
# Process uploaded files
if uploaded_files:
new_items_count = 0
existing_names = set((item['name'], item['type']) for item in st.session_state['items'])
for file in uploaded_files:
if file.type.startswith('image/'):
key = (file.name, 'image')
if key not in existing_names:
img = Image.open(file).convert('RGB')
st.session_state['items'].append({'type': 'image', 'name': file.name, 'img': img, 'emb': None})
new_items_count += 1
elif file.type == 'application/pdf':
pdf_bytes = file.read()
pages = pdf_to_images(pdf_bytes)
for i, page_img in enumerate(pages):
name = f"{file.name} - Page {i+1}"
key = (name, 'pdf_page')
if key not in existing_names:
st.session_state['items'].append({'type': 'pdf_page', 'name': name, 'img': page_img, 'emb': None})
new_items_count += 1
st.success(f"Uploaded {len(uploaded_files)} file(s) with {new_items_count} total items.")
# Sidebar - Content Preview
st.sidebar.header("π Loaded Content")
if st.session_state['items']:
st.sidebar.markdown(f"**Total items:** {len(st.session_state['items'])}")
preview_items = st.session_state['items'][:6] # Show first 6 items
for item in preview_items:
with st.sidebar.container():
small_img = item['img'].resize((120, 120), Image.Resampling.LANCZOS)
caption = item['name'][:15] + "..." if len(item['name']) > 15 else item['name']
st.image(small_img, caption=caption, width=120)
if len(st.session_state['items']) > 6:
st.sidebar.markdown(f"*... and {len(st.session_state['items']) - 6} more items*")
else:
st.sidebar.info("No content loaded yet.")
# Main Interface - Chat
st.subheader("Chat with Your Visual Data")
# Display conversation history
if st.session_state['conversation']:
st.markdown("### Conversation History")
for exchange in st.session_state['conversation']:
with st.expander(f"Q: {exchange['question']} ({exchange['timestamp']})", expanded=False):
st.markdown(f"**Question:** {exchange['question']}")
st.markdown(f"**Answer:** {exchange['answer']}")
st.markdown(f"**Relevant Image:** {exchange['relevant_image']}")
if exchange.get('image_display'):
st.image(exchange['image_display'], caption=exchange['relevant_image'], use_container_width=True)
# Clear conversation button
if st.session_state['conversation']:
if st.button("ποΈ Clear Conversation"):
st.session_state['conversation'] = []
st.rerun()
# Question input form
with st.form("question_form"):
question = st.text_input("Ask a question about your visual data:")
submit_button = st.form_submit_button("Send", type="primary")
# Process question submission
if submit_button:
if not cohere_api or not mistral_api:
st.error("Please provide both Cohere and Mistral API keys.")
elif not question:
st.error("Please enter a question.")
elif not st.session_state['items']:
st.error("No content loaded to search.")
else:
# Compute embeddings and find relevant image
with st.spinner("Computing embeddings and searching..."):
# Generate embeddings for all items if not already done
for item in st.session_state['items']:
if item['emb'] is None:
if item['type'] in ['image', 'pdf_page']:
img_bytes = image_to_bytes(item['img'])
item['emb'] = get_cohere_embedding(cohere_api, img_bytes, input_type='image')
# Get question embedding and find most similar image
q_emb = get_cohere_embedding(cohere_api, question, input_type='text')
emb_list = [item['emb'] for item in st.session_state['items']]
idx, sim = find_most_similar(q_emb, emb_list)
best_item = st.session_state['items'][idx]
# Generate answer using Mistral
st.info("Generating answer with Mistral Vision...")
img_bytes = image_to_bytes(best_item['img'])
answer = mistral_vqa(mistral_api, img_bytes, question)
# Add to conversation history
timestamp = datetime.datetime.now().strftime("%H:%M:%S")
conversation_entry = {
'question': question,
'answer': answer,
'relevant_image': best_item['name'],
'timestamp': timestamp,
'image_display': best_item['img'],
'similarity': sim
}
st.session_state['conversation'].append(conversation_entry)
st.rerun()