Skip to content

A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.

feat: add Image Question Answering using Gemma 4 and PyMuPDF

Tiioluwani2eaf57b71143
diff --git a/multimodal/image_question_answering/.env.example b/multimodal/image_question_answering/.env.example
new file mode 100644
index 0000000..fa40a94
--- /dev/null
+++ b/multimodal/image_question_answering/.env.example
@@ -0,0 +1 @@
+GOOGLE_API_KEY=your_api_key_here
diff --git a/multimodal/image_question_answering/README.md b/multimodal/image_question_answering/README.md
new file mode 100644
index 0000000..8d50626
--- /dev/null
+++ b/multimodal/image_question_answering/README.md
@@ -0,0 +1,105 @@
+# Image Question Answering
+
+> Ask visual questions about any page of a PDF document, answered by Gemma 4 with thinking mode.
+
+## Overview
+
+Upload a PDF, select a page, and ask a natural-language question about its visual content. PyMuPDF renders each page into a high-quality image, which is sent alongside your question to Gemma 4 (`gemma-4-31b-it`) via the Gemini API. The model reasons over the rendered image — including charts, tables, diagrams, and figures — and returns a grounded answer in the Gradio interface.
+
+## Demo
+
+![Demo](assets/demo.png)
+
+## Features
+
+- Renders every PDF page to a full-resolution image using PyMuPDF
+- Page slider lets you navigate and preview any page before asking
+- Answers visual questions grounded in the actual rendered content (charts, tables, figures)
+- Gemma 4 thinking mode (`high`) for multi-step reasoning over complex layouts
+- Clean two-column Gradio UI — page viewer on the left, Q&A on the right
+
+## Tech Stack
+
+| Layer | Library |
+|---|---|
+| PDF to Image | PyMuPDF (`fitz`) |
+| LLM | Gemma 4 (`gemma-4-31b-it`) via Gemini API |
+| SDK | `google-genai` |
+| UI | Gradio |
+
+## Prerequisites
+
+- Python 3.10 or higher
+- A Google AI Studio API key with access to Gemma 4
+
+## Installation
+
+**1. Clone the repository**
+
+```bash
+git clone https://github.com/Sumanth077/Hands-On-AI-Engineering.git
+cd Hands-On-AI-Engineering/multimodal/image_question_answering
+```
+
+**2. Create and activate a virtual environment**
+
+Windows:
+```bash
+python -m venv .venv
+.venv\Scripts\activate
+```
+
+macOS/Linux:
+```bash
+python -m venv .venv
+source .venv/bin/activate
+```
+
+**3. Install dependencies**
+
+```bash
+pip install -r requirements.txt
+```
+
+**4. Configure environment variables**
+
+```bash
+cp .env.example .env
+```
+
+Open `.env` and fill in your API key (see [Environment Variables](#environment-variables)).
+
+## Usage
+
+```bash
+python app.py
+```
+
+Gradio will print a local URL (e.g., `http://127.0.0.1:7860`). Open it in your browser, upload a PDF, select a page with the slider, type your question, and click **Answer Question**.
+
+Example questions:
+- What does the chart on this page show?
+- What is the title of the document?
+- List the bullet points in section 2.
+
+## Environment Variables
+
+Create a `.env` file in the project root with the following key:
+
+| Variable | Description |
+|---|---|
+| `GOOGLE_API_KEY` | Gemini API key — obtain one at [aistudio.google.com](https://aistudio.google.com) |
+
+```env
+GOOGLE_API_KEY=your_api_key_here
+```
+
+## Project Structure
+
+```text
+image-question-answering/
+├── app.py # Gradio app — PDF rendering, Gemma 4 inference, UI
+├── .env # Local secrets (not committed)
+├── .env.example # Environment variable template
+└── requirements.txt
+```
diff --git a/multimodal/image_question_answering/app.py b/multimodal/image_question_answering/app.py
new file mode 100644
index 0000000..1919fb8
--- /dev/null
+++ b/multimodal/image_question_answering/app.py
@@ -0,0 +1,195 @@
+import io
+import os
+
+import fitz # PyMuPDF
+import gradio as gr
+from dotenv import load_dotenv
+from google import genai
+from google.genai import types
+from PIL import Image
+
+# ── environment ───────────────────────────────────────────────────────────────
+
+load_dotenv()
+
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY", "")
+MODEL_ID = "gemma-4-31b-it"
+
+client = genai.Client(api_key=GOOGLE_API_KEY or None)
+
+# ── PDF → images ──────────────────────────────────────────────────────────────
+
+_page_images: list[Image.Image] = []
+
+
+def pdf_to_images(path: str, dpi: int = 150) -> list[Image.Image]:
+ doc = fitz.open(path)
+ zoom = dpi / 72.0
+ mat = fitz.Matrix(zoom, zoom)
+ images: list[Image.Image] = []
+ for page in doc:
+ pix = page.get_pixmap(matrix=mat)
+ images.append(Image.frombytes("RGB", [pix.width, pix.height], pix.samples))
+ doc.close()
+ return images
+
+
+# ── Gradio callbacks ──────────────────────────────────────────────────────────
+
+
+def on_pdf_upload(pdf_file):
+ global _page_images
+
+ if pdf_file is None:
+ _page_images = []
+ return None, gr.update(visible=False, value=1), ""
+
+ _page_images = pdf_to_images(pdf_file.name)
+ n = len(_page_images)
+
+ return (
+ _page_images[0],
+ gr.update(
+ minimum=1,
+ maximum=n,
+ value=1,
+ step=1,
+ label=f"Page (1–{n})",
+ visible=True,
+ ),
+ f"Loaded {n} page{'s' if n != 1 else ''}.",
+ )
+
+
+def on_page_change(page_num: int):
+ idx = int(page_num) - 1
+ if _page_images and 0 <= idx < len(_page_images):
+ return _page_images[idx]
+ return None
+
+
+def on_submit(page_num: int, question: str):
+ if not _page_images:
+ return "Please upload a PDF first."
+ if not question.strip():
+ return "Please type a question."
+
+ idx = int(page_num) - 1
+ if not (0 <= idx < len(_page_images)):
+ return f"Page {int(page_num)} is out of range (PDF has {len(_page_images)} pages)."
+
+ buf = io.BytesIO()
+ _page_images[idx].save(buf, format="PNG")
+ image_bytes = buf.getvalue()
+
+ try:
+ response = client.models.generate_content(
+ model=MODEL_ID,
+ contents=[
+ types.Part.from_bytes(data=image_bytes, mime_type="image/png"),
+ question,
+ ],
+ config=types.GenerateContentConfig(
+ thinking_config=types.ThinkingConfig(thinking_level="high")
+ ),
+ )
+ return response.text or "(no answer returned)"
+ except Exception as exc:
+ return f"Error: {exc}"
+
+
+# ── UI ────────────────────────────────────────────────────────────────────────
+
+with gr.Blocks(title="Image Question Answering") as demo:
+ gr.Markdown(
+ "# Image Question Answering\n"
+ "Upload a PDF, select a page, and ask any visual question — "
+ "answered by **Gemma 4** with thinking mode via the Gemini API."
+ )
+
+ with gr.Row(equal_height=False):
+ # ── left: PDF viewer ──────────────────────────────────────────────
+ with gr.Column(scale=1):
+ pdf_input = gr.File(
+ label="Upload PDF",
+ file_types=[".pdf"],
+ file_count="single",
+ )
+ status_box = gr.Textbox(
+ show_label=False,
+ interactive=False,
+ max_lines=1,
+ container=False,
+ )
+ page_slider = gr.Slider(
+ minimum=1,
+ maximum=2,
+ value=1,
+ step=1,
+ label="Page",
+ visible=False,
+ )
+ page_image = gr.Image(
+ label="Page Preview",
+ type="pil",
+ interactive=False,
+ height=640,
+ )
+
+ # ── right: Q&A ────────────────────────────────────────────────────
+ with gr.Column(scale=1):
+ question_box = gr.Textbox(
+ label="Visual Question",
+ placeholder=(
+ "What does the chart on this page show?\n"
+ "What is the title of the document?\n"
+ "List the bullet points in section 2."
+ ),
+ lines=4,
+ )
+ submit_btn = gr.Button(
+ "Answer Question",
+ variant="primary",
+ size="lg",
+ )
+ answer_box = gr.Textbox(
+ label="Answer",
+ lines=14,
+ interactive=False,
+ )
+
+ # ── event wiring ──────────────────────────────────────────────────────
+ pdf_input.change(
+ fn=on_pdf_upload,
+ inputs=[pdf_input],
+ outputs=[page_image, page_slider, status_box],
+ )
+
+ page_slider.change(
+ fn=on_page_change,
+ inputs=[page_slider],
+ outputs=[page_image],
+ )
+
+ submit_btn.click(
+ fn=on_submit,
+ inputs=[page_slider, question_box],
+ outputs=[answer_box],
+ show_progress="minimal",
+ )
+
+ question_box.submit(
+ fn=on_submit,
+ inputs=[page_slider, question_box],
+ outputs=[answer_box],
+ show_progress="minimal",
+ )
+
+
+if __name__ == "__main__":
+ if not GOOGLE_API_KEY:
+ print(
+ "[WARNING] GOOGLE_API_KEY is not set.\n"
+ "Create a .env file with GOOGLE_API_KEY=<your_key> before running queries."
+ )
+ demo.launch(theme=gr.themes.Soft())
diff --git a/multimodal/image_question_answering/assets/demo.png b/multimodal/image_question_answering/assets/demo.png
new file mode 100644
index 0000000..aab1f2b
Binary files /dev/null and b/multimodal/image_question_answering/assets/demo.png differ
diff --git a/multimodal/image_question_answering/requirements.txt b/multimodal/image_question_answering/requirements.txt
new file mode 100644
index 0000000..394d565
--- /dev/null
+++ b/multimodal/image_question_answering/requirements.txt
@@ -0,0 +1,5 @@
+gradio>=4.44.0
+pymupdf>=1.24.0
+google-genai>=1.0.0
+pillow>=10.0.0
+python-dotenv>=1.0.0