Skip to content

A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.

feat: add Medical Document Parser using Gemma 4 vision and PyMuPDF

Tiioluwanif56c52877dd5
diff --git a/multimodal/medical_document_parser/.env.example b/multimodal/medical_document_parser/.env.example
new file mode 100644
index 0000000..aaf663d
--- /dev/null
+++ b/multimodal/medical_document_parser/.env.example
@@ -0,0 +1 @@
+GOOGLE_API_KEY=your_google_ai_studio_api_key_here
diff --git a/multimodal/medical_document_parser/README.md b/multimodal/medical_document_parser/README.md
new file mode 100644
index 0000000..d3e8358
--- /dev/null
+++ b/multimodal/medical_document_parser/README.md
@@ -0,0 +1,100 @@
+# Medical Document Parser
+
+> Extract a structured clinical profile from medical PDFs and images using Gemma 4 vision.
+
+## Overview
+
+Medical Document Parser is a Gradio app that ingests lab reports, prescriptions, imaging results, and clinical notes. PyMuPDF classifies each PDF page as text or vision, routes content to the appropriate extraction path, and uses Gemma 4 (`gemma-4-31b-it`) via the Google AI Studio API to return a unified JSON clinical profile. Abnormal and critical values are surfaced in a dedicated flagged panel.
+
+## Demo
+
+![Demo](assets/demo.png)
+
+
+## Features
+
+- Upload medical PDFs or images (PNG, JPG, WEBP, BMP, TIFF)
+- Per-page routing: text pages (>50 characters) vs vision pages (charts, scans, complex layouts)
+- Vision pages rendered at 150 DPI with Pillow before analysis
+- Structured JSON extraction with patient info, labs, imaging, and clinical signals
+- Multi-page PDF support with merged results across pages
+- Progress bar during processing
+- Abnormal and critical values highlighted in red
+
+## Tech Stack
+
+| Layer | Technology |
+|-------|------------|
+| LLM | Gemma 4 (`gemma-4-31b-it`) via [Google AI Studio API](https://aistudio.google.com) (`google-genai`, thinking mode) |
+| PDF parsing | PyMuPDF (`fitz`) |
+| Image processing | Pillow |
+| UI | Gradio |
+| Config | `python-dotenv`, Pydantic |
+
+## Prerequisites
+
+- Python 3.10+
+- [Google AI Studio API key](https://aistudio.google.com/app/apikey)
+
+## Installation
+
+```bash
+git clone https://github.com/Sumanth077/Hands-On-AI-Engineering.git
+cd Hands-On-AI-Engineering/multimodal/medical_document_parser
+```
+
+**Windows**
+
+```bash
+py -m venv .venv
+.venv\Scripts\activate
+pip install -r requirements.txt
+copy .env.example .env
+```
+
+**macOS / Linux**
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+cp .env.example .env
+```
+
+Edit `.env` and set your API key before running the app.
+
+## Usage
+
+```bash
+python app.py
+```
+
+Open the local Gradio URL shown in the terminal (typically `http://127.0.0.1:7860`). Upload a medical document, click **Extract Clinical Profile**, and review the JSON output and flagged values.
+
+## Environment Variables
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `GOOGLE_API_KEY` | Yes | API key from [Google AI Studio](https://aistudio.google.com/app/apikey) |
+
+Copy `.env.example` to `.env` and add your key:
+
+```env
+GOOGLE_API_KEY=your_google_ai_studio_api_key_here
+```
+
+## Project Structure
+
+```text
+medical-document-parser/
+├── app.py # Gradio UI and orchestration
+├── document_processor.py # PDF/image page classification and rendering
+├── llm_extractor.py # Gemma 4 API calls and JSON parsing
+├── merger.py # Multi-page result merging
+├── schemas.py # Pydantic clinical profile schema
+├── requirements.txt
+├── .env.example
+├── assets/
+│ └── demo.png # Demo screenshot
+└── README.md
+```
diff --git a/multimodal/medical_document_parser/app.py b/multimodal/medical_document_parser/app.py
new file mode 100644
index 0000000..7536e78
--- /dev/null
+++ b/multimodal/medical_document_parser/app.py
@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import gradio as gr
+from dotenv import load_dotenv
+
+from document_processor import process_upload
+from llm_extractor import _build_client, extract_from_page
+from merger import merge_profiles
+
+PROJECT_DIR = Path(__file__).resolve().parent
+load_dotenv(PROJECT_DIR / ".env")
+
+CUSTOM_CSS = """
+.flagged-box textarea {
+ color: #dc3545 !important;
+ font-weight: 600 !important;
+ background: #fff5f5 !important;
+ border-color: #f5c2c7 !important;
+}
+"""
+
+
+def _format_flagged_text(flagged_items: list[str]) -> str:
+ if not flagged_items:
+ return "No abnormal or critical values flagged."
+ return "\n".join(f"• {item}" for item in flagged_items)
+
+
+def parse_document(
+ upload: str | None,
+ progress: gr.Progress = gr.Progress(),
+) -> tuple[dict, str]:
+ if upload is None:
+ raise gr.Error("Please upload a medical PDF or image.")
+
+ file_path = Path(upload)
+ if not file_path.exists():
+ raise gr.Error("Uploaded file could not be found.")
+
+ progress(0, desc="Preparing document...")
+ pages = process_upload(file_path)
+ if not pages:
+ raise gr.Error("No pages were found in the uploaded document.")
+
+ client = _build_client()
+ profiles = []
+ total = len(pages)
+
+ for index, page in enumerate(pages, start=1):
+ progress(
+ index / total,
+ desc=f"Analyzing page {index}/{total} ({page.kind}) with Gemma 4...",
+ )
+ profiles.append(extract_from_page(client, page))
+
+ progress(1.0, desc="Merging results...")
+ merged = merge_profiles(profiles)
+ payload = merged.model_dump()
+ return payload, _format_flagged_text(payload["flagged_items"])
+
+
+def build_app() -> gr.Blocks:
+ with gr.Blocks(title="Medical Document Parser") as demo:
+ gr.Markdown(
+ """
+ # Medical Document Parser
+ Upload a medical **PDF** or **image** (lab report, prescription, imaging result, or clinical notes).
+ The app routes each page through text or vision extraction, then uses **Gemma 4** to build a unified clinical profile.
+ """
+ )
+
+ with gr.Row():
+ upload = gr.File(
+ label="Medical document",
+ file_types=[".pdf", ".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"],
+ type="filepath",
+ )
+
+ parse_button = gr.Button("Extract Clinical Profile", variant="primary")
+
+ with gr.Row():
+ json_output = gr.JSON(label="Structured Clinical Profile")
+ flagged_output = gr.Textbox(
+ label="Flagged Abnormal Values",
+ lines=10,
+ elem_classes=["flagged-box"],
+ )
+
+ parse_button.click(
+ fn=parse_document,
+ inputs=[upload],
+ outputs=[json_output, flagged_output],
+ )
+
+ gr.Markdown(
+ """
+ **How it works**
+ 1. PyMuPDF classifies each PDF page as text (>50 characters) or vision.
+ 2. Text pages are sent directly to Gemma 4; vision pages are rendered at 150 DPI.
+ 3. Per-page extractions are merged into one JSON profile with abnormal values highlighted.
+ """
+ )
+
+ return demo
+
+
+if __name__ == "__main__":
+ build_app().launch(css=CUSTOM_CSS, theme=gr.themes.Soft())
diff --git a/multimodal/medical_document_parser/assets/demo.png b/multimodal/medical_document_parser/assets/demo.png
new file mode 100644
index 0000000..bd0e57d
Binary files /dev/null and b/multimodal/medical_document_parser/assets/demo.png differ
diff --git a/multimodal/medical_document_parser/document_processor.py b/multimodal/medical_document_parser/document_processor.py
new file mode 100644
index 0000000..c6a1274
--- /dev/null
+++ b/multimodal/medical_document_parser/document_processor.py
@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+import io
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+import fitz
+from PIL import Image
+
+TEXT_THRESHOLD = 50
+RENDER_DPI = 150
+
+PageKind = Literal["text", "vision"]
+
+
+@dataclass
+class ProcessedPage:
+ page_number: int
+ kind: PageKind
+ text: str | None = None
+ image_bytes: bytes | None = None
+
+
+def _text_page(page: fitz.Page) -> ProcessedPage:
+ return ProcessedPage(
+ page_number=page.number + 1,
+ kind="text",
+ text=page.get_text("text").strip(),
+ )
+
+
+def _vision_page(page: fitz.Page) -> ProcessedPage:
+ pixmap = page.get_pixmap(dpi=RENDER_DPI)
+ image = Image.frombytes("RGB", (pixmap.width, pixmap.height), pixmap.samples)
+ buffer = io.BytesIO()
+ image.save(buffer, format="PNG")
+ return ProcessedPage(
+ page_number=page.number + 1,
+ kind="vision",
+ image_bytes=buffer.getvalue(),
+ )
+
+
+def _classify_pdf_page(page: fitz.Page) -> ProcessedPage:
+ text = page.get_text("text").strip()
+ if len(text) > TEXT_THRESHOLD:
+ return ProcessedPage(
+ page_number=page.number + 1,
+ kind="text",
+ text=text,
+ )
+ return _vision_page(page)
+
+
+def _image_to_page(image_bytes: bytes, page_number: int = 1) -> ProcessedPage:
+ image = Image.open(io.BytesIO(image_bytes))
+ if image.mode != "RGB":
+ image = image.convert("RGB")
+ buffer = io.BytesIO()
+ image.save(buffer, format="PNG")
+ return ProcessedPage(
+ page_number=page_number,
+ kind="vision",
+ image_bytes=buffer.getvalue(),
+ )
+
+
+def process_upload(file_path: str | Path) -> list[ProcessedPage]:
+ path = Path(file_path)
+ suffix = path.suffix.lower()
+
+ if suffix == ".pdf":
+ pages: list[ProcessedPage] = []
+ with fitz.open(path) as document:
+ for page in document:
+ pages.append(_classify_pdf_page(page))
+ return pages
+
+ if suffix in {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"}:
+ return [_image_to_page(path.read_bytes())]
+
+ raise ValueError(f"Unsupported file type: {suffix or 'unknown'}")
diff --git a/multimodal/medical_document_parser/llm_extractor.py b/multimodal/medical_document_parser/llm_extractor.py
new file mode 100644
index 0000000..1397130
--- /dev/null
+++ b/multimodal/medical_document_parser/llm_extractor.py
@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+
+from google import genai
+from google.genai import types
+
+from document_processor import ProcessedPage
+from schemas import ClinicalProfile
+
+MODEL_ID = "gemma-4-31b-it"
+
+SYSTEM_INSTRUCTION = """You are a clinical document extraction assistant.
+Extract structured medical information from the provided page content.
+Return only factual information present in the document.
+Use empty strings or empty lists when information is missing.
+For lab findings, set status to:
+- normal: within reference range
+- abnormal: outside reference range but not life-threatening
+- critical: dangerously out of range or marked critical
+Populate flagged_items with concise descriptions of abnormal or critical values."""
+
+EXTRACTION_PROMPT = """Extract a structured clinical profile from this medical document page.
+Include patient demographics, laboratory results, imaging findings, clinical signals,
+and any abnormal or critical values in flagged_items."""
+
+
+def _build_client() -> genai.Client:
+ api_key = os.getenv("GOOGLE_API_KEY")
+ if not api_key:
+ raise ValueError("GOOGLE_API_KEY is not set. Add it to your .env file.")
+ return genai.Client(api_key=api_key)
+
+
+def _generation_config() -> types.GenerateContentConfig:
+ return types.GenerateContentConfig(
+ system_instruction=SYSTEM_INSTRUCTION,
+ response_mime_type="application/json",
+ response_json_schema=ClinicalProfile.model_json_schema(),
+ thinking_config=types.ThinkingConfig(
+ thinking_level=types.ThinkingLevel.HIGH,
+ ),
+ )
+
+
+def _parse_response_text(text: str) -> ClinicalProfile:
+ cleaned = text.strip()
+ if cleaned.startswith("```"):
+ cleaned = re.sub(r"^```(?:json)?\s*", "", cleaned)
+ cleaned = re.sub(r"\s*```$", "", cleaned)
+ data = json.loads(cleaned)
+ return ClinicalProfile.model_validate(data)
+
+
+def extract_from_page(client: genai.Client, page: ProcessedPage) -> ClinicalProfile:
+ config = _generation_config()
+ page_context = f"{EXTRACTION_PROMPT}\n\nPage {page.page_number}."
+
+ if page.kind == "text" and page.text:
+ response = client.models.generate_content(
+ model=MODEL_ID,
+ contents=f"{page_context}\n\nDocument text:\n{page.text}",
+ config=config,
+ )
+ elif page.kind == "vision" and page.image_bytes:
+ response = client.models.generate_content(
+ model=MODEL_ID,
+ contents=[
+ types.Part.from_bytes(data=page.image_bytes, mime_type="image/png"),
+ page_context,
+ ],
+ config=config,
+ )
+ else:
+ return ClinicalProfile()
+
+ if not response.text:
+ return ClinicalProfile()
+
+ return _parse_response_text(response.text)
diff --git a/multimodal/medical_document_parser/merger.py b/multimodal/medical_document_parser/merger.py
new file mode 100644
index 0000000..00aae86
--- /dev/null
+++ b/multimodal/medical_document_parser/merger.py
@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from schemas import ClinicalProfile, ImagingResult, LabFinding, PatientInfo
+
+
+def _merge_patient(current: PatientInfo, incoming: PatientInfo) -> PatientInfo:
+ return PatientInfo(
+ name=current.name or incoming.name,
+ age=current.age or incoming.age,
+ date=current.date or incoming.date,
+ )
+
+
+def _dedupe_lab_findings(findings: list[LabFinding]) -> list[LabFinding]:
+ seen: dict[str, LabFinding] = {}
+ for finding in findings:
+ key = finding.test.strip().lower()
+ if not key:
+ continue
+ existing = seen.get(key)
+ if existing is None or (existing.status == "normal" and finding.status != "normal"):
+ seen[key] = finding
+ return list(seen.values())
+
+
+def _dedupe_strings(items: list[str]) -> list[str]:
+ seen: set[str] = set()
+ result: list[str] = []
+ for item in items:
+ normalized = item.strip()
+ if not normalized:
+ continue
+ key = normalized.lower()
+ if key in seen:
+ continue
+ seen.add(key)
+ result.append(normalized)
+ return result
+
+
+def merge_profiles(profiles: list[ClinicalProfile]) -> ClinicalProfile:
+ merged = ClinicalProfile()
+
+ for profile in profiles:
+ merged.patient = _merge_patient(merged.patient, profile.patient)
+ merged.lab_findings.extend(profile.lab_findings)
+ merged.imaging_results.extend(profile.imaging_results)
+ merged.clinical_signals.extend(profile.clinical_signals)
+ merged.flagged_items.extend(profile.flagged_items)
+
+ merged.lab_findings = _dedupe_lab_findings(merged.lab_findings)
+ merged.clinical_signals = _dedupe_strings(merged.clinical_signals)
+ merged.flagged_items = _dedupe_strings(merged.flagged_items)
+
+ for finding in merged.lab_findings:
+ if finding.status in {"abnormal", "critical"}:
+ label = f"{finding.test}: {finding.value}"
+ if finding.normal_range:
+ label += f" (ref: {finding.normal_range})"
+ label += f" [{finding.status}]"
+ if label not in merged.flagged_items:
+ merged.flagged_items.append(label)
+
+ return merged
diff --git a/multimodal/medical_document_parser/requirements.txt b/multimodal/medical_document_parser/requirements.txt
new file mode 100644
index 0000000..fc5b7ae
--- /dev/null
+++ b/multimodal/medical_document_parser/requirements.txt
@@ -0,0 +1,6 @@
+gradio>=5.0.0
+google-genai>=1.0.0
+PyMuPDF>=1.24.0
+Pillow>=10.0.0
+python-dotenv>=1.0.0
+pydantic>=2.0.0
diff --git a/multimodal/medical_document_parser/schemas.py b/multimodal/medical_document_parser/schemas.py
new file mode 100644
index 0000000..8b2016e
--- /dev/null
+++ b/multimodal/medical_document_parser/schemas.py
@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class PatientInfo(BaseModel):
+ name: str = ""
+ age: str = ""
+ date: str = ""
+
+
+class LabFinding(BaseModel):
+ test: str = ""
+ value: str = ""
+ normal_range: str = ""
+ status: Literal["normal", "abnormal", "critical"] = "normal"
+
+
+class ImagingResult(BaseModel):
+ study: str = ""
+ findings: str = ""
+ impression: str = ""
+
+
+class ClinicalProfile(BaseModel):
+ patient: PatientInfo = Field(default_factory=PatientInfo)
+ lab_findings: list[LabFinding] = Field(default_factory=list)
+ imaging_results: list[ImagingResult] = Field(default_factory=list)
+ clinical_signals: list[str] = Field(default_factory=list)
+ flagged_items: list[str] = Field(default_factory=list)