A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.
feat: add Image Question Answering using Gemma 4 and PyMuPDF
Tiioluwani2eaf57b71143
diff --git a/multimodal/image_question_answering/.env.example b/multimodal/image_question_answering/.env.examplenew file mode 100644index 0000000..fa40a94--- /dev/null+++ b/multimodal/image_question_answering/.env.example@@ -0,0 +1 @@+GOOGLE_API_KEY=your_api_key_herediff --git a/multimodal/image_question_answering/README.md b/multimodal/image_question_answering/README.mdnew file mode 100644index 0000000..8d50626--- /dev/null+++ b/multimodal/image_question_answering/README.md@@ -0,0 +1,105 @@+# Image Question Answering++> Ask visual questions about any page of a PDF document, answered by Gemma 4 with thinking mode.++## Overview++Upload a PDF, select a page, and ask a natural-language question about its visual content. PyMuPDF renders each page into a high-quality image, which is sent alongside your question to Gemma 4 (`gemma-4-31b-it`) via the Gemini API. The model reasons over the rendered image — including charts, tables, diagrams, and figures — and returns a grounded answer in the Gradio interface.++## Demo++++## Features++- Renders every PDF page to a full-resolution image using PyMuPDF+- Page slider lets you navigate and preview any page before asking+- Answers visual questions grounded in the actual rendered content (charts, tables, figures)+- Gemma 4 thinking mode (`high`) for multi-step reasoning over complex layouts+- Clean two-column Gradio UI — page viewer on the left, Q&A on the right++## Tech Stack++| Layer | Library |+|---|---|+| PDF to Image | PyMuPDF (`fitz`) |+| LLM | Gemma 4 (`gemma-4-31b-it`) via Gemini API |+| SDK | `google-genai` |+| UI | Gradio |++## Prerequisites++- Python 3.10 or higher+- A Google AI Studio API key with access to Gemma 4++## Installation++**1. Clone the repository**++```bash+git clone https://github.com/Sumanth077/Hands-On-AI-Engineering.git+cd Hands-On-AI-Engineering/multimodal/image_question_answering+```++**2. Create and activate a virtual environment**++Windows:+```bash+python -m venv .venv+.venv\Scripts\activate+```++macOS/Linux:+```bash+python -m venv .venv+source .venv/bin/activate+```++**3. Install dependencies**++```bash+pip install -r requirements.txt+```++**4. Configure environment variables**++```bash+cp .env.example .env+```++Open `.env` and fill in your API key (see [Environment Variables](#environment-variables)).++## Usage++```bash+python app.py+```++Gradio will print a local URL (e.g., `http://127.0.0.1:7860`). Open it in your browser, upload a PDF, select a page with the slider, type your question, and click **Answer Question**.++Example questions:+- What does the chart on this page show?+- What is the title of the document?+- List the bullet points in section 2.++## Environment Variables++Create a `.env` file in the project root with the following key:++| Variable | Description |+|---|---|+| `GOOGLE_API_KEY` | Gemini API key — obtain one at [aistudio.google.com](https://aistudio.google.com) |++```env+GOOGLE_API_KEY=your_api_key_here+```++## Project Structure++```text+image-question-answering/+├── app.py # Gradio app — PDF rendering, Gemma 4 inference, UI+├── .env # Local secrets (not committed)+├── .env.example # Environment variable template+└── requirements.txt+```diff --git a/multimodal/image_question_answering/app.py b/multimodal/image_question_answering/app.pynew file mode 100644index 0000000..1919fb8--- /dev/null+++ b/multimodal/image_question_answering/app.py@@ -0,0 +1,195 @@+import io+import os++import fitz # PyMuPDF+import gradio as gr+from dotenv import load_dotenv+from google import genai+from google.genai import types+from PIL import Image++# ── environment ───────────────────────────────────────────────────────────────++load_dotenv()++GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY", "")+MODEL_ID = "gemma-4-31b-it"++client = genai.Client(api_key=GOOGLE_API_KEY or None)++# ── PDF → images ──────────────────────────────────────────────────────────────++_page_images: list[Image.Image] = []+++def pdf_to_images(path: str, dpi: int = 150) -> list[Image.Image]:+ doc = fitz.open(path)+ zoom = dpi / 72.0+ mat = fitz.Matrix(zoom, zoom)+ images: list[Image.Image] = []+ for page in doc:+ pix = page.get_pixmap(matrix=mat)+ images.append(Image.frombytes("RGB", [pix.width, pix.height], pix.samples))+ doc.close()+ return images+++# ── Gradio callbacks ──────────────────────────────────────────────────────────+++def on_pdf_upload(pdf_file):+ global _page_images++ if pdf_file is None:+ _page_images = []+ return None, gr.update(visible=False, value=1), ""++ _page_images = pdf_to_images(pdf_file.name)+ n = len(_page_images)++ return (+ _page_images[0],+ gr.update(+ minimum=1,+ maximum=n,+ value=1,+ step=1,+ label=f"Page (1–{n})",+ visible=True,+ ),+ f"Loaded {n} page{'s' if n != 1 else ''}.",+ )+++def on_page_change(page_num: int):+ idx = int(page_num) - 1+ if _page_images and 0 <= idx < len(_page_images):+ return _page_images[idx]+ return None+++def on_submit(page_num: int, question: str):+ if not _page_images:+ return "Please upload a PDF first."+ if not question.strip():+ return "Please type a question."++ idx = int(page_num) - 1+ if not (0 <= idx < len(_page_images)):+ return f"Page {int(page_num)} is out of range (PDF has {len(_page_images)} pages)."++ buf = io.BytesIO()+ _page_images[idx].save(buf, format="PNG")+ image_bytes = buf.getvalue()++ try:+ response = client.models.generate_content(+ model=MODEL_ID,+ contents=[+ types.Part.from_bytes(data=image_bytes, mime_type="image/png"),+ question,+ ],+ config=types.GenerateContentConfig(+ thinking_config=types.ThinkingConfig(thinking_level="high")+ ),+ )+ return response.text or "(no answer returned)"+ except Exception as exc:+ return f"Error: {exc}"+++# ── UI ────────────────────────────────────────────────────────────────────────++with gr.Blocks(title="Image Question Answering") as demo:+ gr.Markdown(+ "# Image Question Answering\n"+ "Upload a PDF, select a page, and ask any visual question — "+ "answered by **Gemma 4** with thinking mode via the Gemini API."+ )++ with gr.Row(equal_height=False):+ # ── left: PDF viewer ──────────────────────────────────────────────+ with gr.Column(scale=1):+ pdf_input = gr.File(+ label="Upload PDF",+ file_types=[".pdf"],+ file_count="single",+ )+ status_box = gr.Textbox(+ show_label=False,+ interactive=False,+ max_lines=1,+ container=False,+ )+ page_slider = gr.Slider(+ minimum=1,+ maximum=2,+ value=1,+ step=1,+ label="Page",+ visible=False,+ )+ page_image = gr.Image(+ label="Page Preview",+ type="pil",+ interactive=False,+ height=640,+ )++ # ── right: Q&A ────────────────────────────────────────────────────+ with gr.Column(scale=1):+ question_box = gr.Textbox(+ label="Visual Question",+ placeholder=(+ "What does the chart on this page show?\n"+ "What is the title of the document?\n"+ "List the bullet points in section 2."+ ),+ lines=4,+ )+ submit_btn = gr.Button(+ "Answer Question",+ variant="primary",+ size="lg",+ )+ answer_box = gr.Textbox(+ label="Answer",+ lines=14,+ interactive=False,+ )++ # ── event wiring ──────────────────────────────────────────────────────+ pdf_input.change(+ fn=on_pdf_upload,+ inputs=[pdf_input],+ outputs=[page_image, page_slider, status_box],+ )++ page_slider.change(+ fn=on_page_change,+ inputs=[page_slider],+ outputs=[page_image],+ )++ submit_btn.click(+ fn=on_submit,+ inputs=[page_slider, question_box],+ outputs=[answer_box],+ show_progress="minimal",+ )++ question_box.submit(+ fn=on_submit,+ inputs=[page_slider, question_box],+ outputs=[answer_box],+ show_progress="minimal",+ )+++if __name__ == "__main__":+ if not GOOGLE_API_KEY:+ print(+ "[WARNING] GOOGLE_API_KEY is not set.\n"+ "Create a .env file with GOOGLE_API_KEY=<your_key> before running queries."+ )+ demo.launch(theme=gr.themes.Soft())diff --git a/multimodal/image_question_answering/assets/demo.png b/multimodal/image_question_answering/assets/demo.pngnew file mode 100644index 0000000..aab1f2bBinary files /dev/null and b/multimodal/image_question_answering/assets/demo.png differdiff --git a/multimodal/image_question_answering/requirements.txt b/multimodal/image_question_answering/requirements.txtnew file mode 100644index 0000000..394d565--- /dev/null+++ b/multimodal/image_question_answering/requirements.txt@@ -0,0 +1,5 @@+gradio>=4.44.0+pymupdf>=1.24.0+google-genai>=1.0.0+pillow>=10.0.0+python-dotenv>=1.0.0