A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.
feat: add Medical Prescription Digitizer using Mistral Large 3 and RxNorm
Tiioluwanif186efbbb73f
diff --git a/OCR/medical_prescription_digitizer/.env.example b/OCR/medical_prescription_digitizer/.env.examplenew file mode 100644index 0000000..b263636--- /dev/null+++ b/OCR/medical_prescription_digitizer/.env.example@@ -0,0 +1 @@+MISTRAL_API_KEY=your_mistral_api_key_herediff --git a/OCR/medical_prescription_digitizer/README.md b/OCR/medical_prescription_digitizer/README.mdnew file mode 100644index 0000000..dfe0cdd--- /dev/null+++ b/OCR/medical_prescription_digitizer/README.md@@ -0,0 +1,114 @@+# Medical Prescription Digitizer++> Extract, structure, and validate handwritten or printed prescriptions using Mistral Large 3 and RxNorm.++## Overview++Upload a prescription image — handwritten or printed — and the app uses Mistral Large 3's vision capabilities to read and interpret it, including messy handwriting and medical abbreviations. Extracted drug names are then validated against the [RxNorm](https://rxnav.nlm.nih.gov/) drug database (free, no API key required). Validated drugs are highlighted in green; unrecognised or flagged ones appear in red.++## Features++- Reads handwritten and printed prescriptions via multimodal LLM+- Decodes standard medical abbreviations (QD, BID, TID, PRN, PO, etc.)+- Extracts patient name, doctor name, date, medications with dosage and frequency, and additional notes+- Flags illegible or uncertain fields instead of guessing+- Validates every drug name against the RxNorm API in real time+- Displays structured results with per-medication validation status and RxNorm IDs++## Tech Stack++| Layer | Technology |+|---|---|+| Model | Mistral Large 3 (`mistral-large-latest`) |+| Structured outputs | `mistralai` SDK `chat.parse` + Pydantic v2 |+| Schema validation | Pydantic v2 |+| Drug validation | RxNorm API (free, no key needed) |+| UI | Streamlit |++## Prerequisites++- Python 3.10 or higher+- A Mistral API key — get one at [platform.mistral.ai](https://platform.mistral.ai)++## Installation++```bash+git clone https://github.com/Sumanth077/Hands-On-AI-Engineering.git+cd Hands-On-AI-Engineering/ocr/medical-prescription-digitizer+```++**Windows**++```bash+python -m venv .venv+.venv\Scripts\activate+pip install -r requirements.txt+```++**macOS / Linux**++```bash+python -m venv .venv+source .venv/bin/activate+pip install -r requirements.txt+```++Copy the example env file and add your API key:++```bash+cp .env.example .env+```++## Usage++```bash+streamlit run app.py+```++Open `http://localhost:8501` in your browser, enter your Mistral API key in the sidebar, upload a prescription image (JPG, PNG, or WEBP), then click **Digitize Prescription**.++**Example input:** a handwritten prescription image containing:++```text+Patient: John Smith Date: 04/18/2026+Dr. Sarah Lee, MD++Rx:+1. Amoxicillin 500mg — TID x 7 days+2. Ibuprofen 400mg — BID PRN pain+3. [illegible] 10mg — QD+```++**Structured output:**++```text+Patient John Smith+Doctor Dr. Sarah Lee+Date 04/18/2026++Medications+✅ Amoxicillin 500 mg · three times daily · 7 days [RxNorm ID: 723]+✅ Ibuprofen 400 mg · twice daily · as needed [RxNorm ID: 5640]+⚠️ [illegible] 10 mg · once daily Drug name not found in RxNorm — possible misread++Illegible fields+• Medication 3 drug name unclear+```++## Environment Variables++| Variable | Description |+|---|---|+| `MISTRAL_API_KEY` | Your Mistral API key from [platform.mistral.ai](https://platform.mistral.ai) |++## Project Structure++```text+medical-prescription-digitizer/+├── app.py # Streamlit UI — upload, display, and orchestration+├── extractor.py # Sends image to Mistral Large 3 and returns a Prescription object+├── schemas.py # Pydantic models: Prescription and Medication+├── validator.py # Queries RxNorm API to validate each drug name+├── requirements.txt # Python dependencies+└── .env.example # Environment variable template+```diff --git a/OCR/medical_prescription_digitizer/app.py b/OCR/medical_prescription_digitizer/app.pynew file mode 100644index 0000000..3ea3aea--- /dev/null+++ b/OCR/medical_prescription_digitizer/app.py@@ -0,0 +1,188 @@+import os+import io++import streamlit as st+from dotenv import load_dotenv+from PIL import Image++from extractor import extract_prescription+from validator import validate_prescription_drugs+from schemas import Prescription++load_dotenv()++st.set_page_config(+ page_title="Medical Prescription Digitizer",+ page_icon="💊",+ layout="wide",+)++# ── Styles ────────────────────────────────────────────────────────────────────+st.markdown(+ """+ <style>+ .valid-drug { background:#d4edda; border-left:4px solid #28a745;+ padding:10px 14px; border-radius:6px; margin-bottom:8px; }+ .invalid-drug { background:#f8d7da; border-left:4px solid #dc3545;+ padding:10px 14px; border-radius:6px; margin-bottom:8px; }+ .field-row { display:flex; gap:12px; flex-wrap:wrap; margin-top:4px; }+ .field-chip { background:#e9ecef; border-radius:12px;+ padding:2px 10px; font-size:0.85rem; color:#495057; }+ .illegible-box{ background:#fff3cd; border-left:4px solid #ffc107;+ padding:10px 14px; border-radius:6px; margin-top:8px; }+ .section-card { background:#f8f9fa; border-radius:10px;+ padding:18px 22px; margin-bottom:16px; }+ </style>+ """,+ unsafe_allow_html=True,+)++# ── Header ────────────────────────────────────────────────────────────────────+st.title("💊 Medical Prescription Digitizer")+st.caption("Upload a prescription image — handwritten or printed — to extract and validate its contents.")+st.divider()+++# ── Sidebar: API key ──────────────────────────────────────────────────────────+with st.sidebar:+ st.header("⚙️ Configuration")+ api_key = st.text_input(+ "Mistral API Key",+ value=os.getenv("MISTRAL_API_KEY", ""),+ type="password",+ help="Your Mistral API key. Store it in .env as MISTRAL_API_KEY to avoid re-entering.",+ )+ st.markdown("---")+ st.markdown("**How it works**")+ st.markdown(+ "1. Upload a prescription image\n"+ "2. Mistral Large 3 reads and interprets it\n"+ "3. Drug names are validated via [RxNorm](https://rxnav.nlm.nih.gov/)\n"+ "4. Results are displayed with validation status"+ )+ st.markdown("---")+ st.markdown("**Supported formats:** JPG, PNG, WEBP")+++# ── Upload ────────────────────────────────────────────────────────────────────+uploaded_file = st.file_uploader(+ "Upload Prescription Image",+ type=["jpg", "jpeg", "png", "webp"],+ help="Upload a clear photo of a handwritten or printed prescription.",+)++if uploaded_file:+ col_img, col_info = st.columns([1, 1], gap="large")++ with col_img:+ st.subheader("Uploaded Image")+ image = Image.open(uploaded_file)+ st.image(image, use_container_width=True)+ uploaded_file.seek(0)++ with col_info:+ st.subheader("Extraction & Validation")++ if not api_key:+ st.error("Please enter your Mistral API key in the sidebar to proceed.")+ st.stop()++ if st.button("🔍 Digitize Prescription", type="primary", use_container_width=True):+ prescription: Prescription | None = None++ with st.spinner("Reading prescription with Mistral Large 3…"):+ try:+ prescription = extract_prescription(uploaded_file, api_key)+ except Exception as e:+ st.error(f"Extraction failed: {e}")+ st.stop()++ with st.spinner("Validating drug names against RxNorm…"):+ try:+ prescription = validate_prescription_drugs(prescription)+ except Exception as e:+ st.warning(f"Validation step encountered an error: {e}")++ # ── Display results ───────────────────────────────────────────────+ st.success("Prescription processed successfully!")++ # Patient / Doctor / Date+ st.markdown('<div class="section-card">', unsafe_allow_html=True)+ meta_cols = st.columns(3)+ with meta_cols[0]:+ st.metric("Patient", prescription.patient_name or "—")+ with meta_cols[1]:+ st.metric("Doctor", prescription.doctor_name or "—")+ with meta_cols[2]:+ st.metric("Date", prescription.date or "—")+ st.markdown("</div>", unsafe_allow_html=True)++ # Medications+ st.markdown("#### Medications")++ if not prescription.medications:+ st.warning("No medications could be extracted from this prescription.")+ else:+ validated_count = sum(1 for m in prescription.medications if m.is_validated)+ total = len(prescription.medications)+ st.caption(f"{validated_count}/{total} drug names validated against RxNorm")++ for med in prescription.medications:+ css_class = "valid-drug" if med.is_validated else "invalid-drug"+ status_icon = "✅" if med.is_validated else "⚠️"++ chips = ""+ if med.dosage:+ chips += f'<span class="field-chip">💊 {med.dosage}</span>'+ if med.frequency:+ chips += f'<span class="field-chip">🕐 {med.frequency}</span>'+ if med.duration:+ chips += f'<span class="field-chip">📅 {med.duration}</span>'++ note_html = ""+ if med.validation_note:+ note_color = "#155724" if med.is_validated else "#721c24"+ note_html = f'<div style="font-size:0.8rem;color:{note_color};margin-top:4px">{med.validation_note}</div>'++ st.markdown(+ f"""+ <div class="{css_class}">+ <strong>{status_icon} {med.drug_name}</strong>+ <div class="field-row">{chips}</div>+ {note_html}+ </div>+ """,+ unsafe_allow_html=True,+ )++ # Notes+ if prescription.notes:+ st.markdown("#### Additional Notes")+ st.info(prescription.notes)++ # Illegible fields+ if prescription.illegible_fields:+ st.markdown(+ '<div class="illegible-box">'+ "<strong>⚠️ Illegible / Uncertain Fields</strong><br>"+ + "<br>".join(f"• {f}" for f in prescription.illegible_fields)+ + "</div>",+ unsafe_allow_html=True,+ )++ # Raw JSON expander+ with st.expander("🗂️ View Raw Extracted JSON"):+ st.json(prescription.model_dump())++else:+ # Empty state+ st.markdown(+ """+ <div style="text-align:center;padding:60px 20px;color:#6c757d;">+ <div style="font-size:4rem">📋</div>+ <h3>No prescription uploaded yet</h3>+ <p>Upload a JPG, PNG, or WEBP image of a prescription above to get started.</p>+ </div>+ """,+ unsafe_allow_html=True,+ )diff --git a/OCR/medical_prescription_digitizer/extractor.py b/OCR/medical_prescription_digitizer/extractor.pynew file mode 100644index 0000000..5746618--- /dev/null+++ b/OCR/medical_prescription_digitizer/extractor.py@@ -0,0 +1,73 @@+import base64+import io++from mistralai.client import Mistral+from PIL import Image++from schemas import Prescription+++SYSTEM_PROMPT = """You are a medical prescription digitizer. Your task is to carefully read and extract+structured information from prescription images, including handwritten ones.++Guidelines:+- Extract ALL medications listed, even if handwriting is difficult+- Decode common medical abbreviations: QD/OD=once daily, BID=twice daily, TID=three times daily,+ QID=four times daily, PRN=as needed, PO=by mouth, SIG=directions, Rx=prescription+- For illegible sections, add a descriptive entry to illegible_fields (e.g. "medication 2 dosage unclear")+- Do NOT guess drug names if truly illegible — mark them as illegible instead+- Extract dosage units precisely: mg, mcg, ml, units, etc.+- If a field is absent from the prescription, leave it as null+"""+++def image_to_base64(image_file) -> tuple[str, str]:+ """Convert uploaded file to base64 string. Returns (base64_data, media_type)."""+ image_bytes = image_file.read()+ image_file.seek(0)++ img = Image.open(io.BytesIO(image_bytes))+ fmt = img.format or "JPEG"+ media_type_map = {+ "JPEG": "image/jpeg",+ "PNG": "image/png",+ "GIF": "image/gif",+ "WEBP": "image/webp",+ }+ media_type = media_type_map.get(fmt, "image/jpeg")++ b64 = base64.b64encode(image_bytes).decode("utf-8")+ return b64, media_type+++def extract_prescription(image_file, api_key: str) -> Prescription:+ """Send image to Mistral Large 3 and extract structured prescription data."""+ b64_data, media_type = image_to_base64(image_file)++ client = Mistral(api_key=api_key)++ result = client.chat.parse(+ response_format=Prescription,+ model="mistral-large-latest",+ messages=[+ {+ "role": "system",+ "content": SYSTEM_PROMPT,+ },+ {+ "role": "user",+ "content": [+ {+ "type": "text",+ "text": "Please extract all structured information from this medical prescription image.",+ },+ {+ "type": "image_url",+ "image_url": f"data:{media_type};base64,{b64_data}",+ },+ ],+ },+ ],+ )++ return result.choices[0].message.parseddiff --git a/OCR/medical_prescription_digitizer/requirements.txt b/OCR/medical_prescription_digitizer/requirements.txtnew file mode 100644index 0000000..5cb64f8--- /dev/null+++ b/OCR/medical_prescription_digitizer/requirements.txt@@ -0,0 +1,6 @@+streamlit>=1.32.0+mistralai>=2.0.0+pydantic>=2.0.0+python-dotenv>=1.0.0+requests>=2.31.0+Pillow>=10.0.0diff --git a/OCR/medical_prescription_digitizer/schemas.py b/OCR/medical_prescription_digitizer/schemas.pynew file mode 100644index 0000000..bf18265--- /dev/null+++ b/OCR/medical_prescription_digitizer/schemas.py@@ -0,0 +1,23 @@+from pydantic import BaseModel, Field+from typing import Optional, List+++class Medication(BaseModel):+ drug_name: str = Field(description="Name of the drug/medication as written on prescription")+ dosage: Optional[str] = Field(default=None, description="Dosage amount and unit, e.g. '500mg', '10mg/5ml'")+ frequency: Optional[str] = Field(default=None, description="How often to take, e.g. 'twice daily', 'every 8 hours', 'TID'")+ duration: Optional[str] = Field(default=None, description="Duration of treatment, e.g. '7 days', '2 weeks'")+ is_validated: bool = Field(default=False, description="Whether drug name was validated against RxNorm")+ validation_note: Optional[str] = Field(default=None, description="Note about validation result")+++class Prescription(BaseModel):+ patient_name: Optional[str] = Field(default=None, description="Full name of the patient")+ doctor_name: Optional[str] = Field(default=None, description="Name of the prescribing doctor")+ date: Optional[str] = Field(default=None, description="Date on the prescription")+ medications: List[Medication] = Field(description="List of all medications prescribed")+ notes: Optional[str] = Field(default=None, description="Any additional notes, instructions, or diagnoses on the prescription")+ illegible_fields: List[str] = Field(+ default=[],+ description="List of fields or sections that were illegible or unclear, e.g. ['patient address', 'drug 2 dosage']"+ )diff --git a/OCR/medical_prescription_digitizer/validator.py b/OCR/medical_prescription_digitizer/validator.pynew file mode 100644index 0000000..f63acf2--- /dev/null+++ b/OCR/medical_prescription_digitizer/validator.py@@ -0,0 +1,52 @@+import requests+from schemas import Medication, Prescription++RXNORM_BASE_URL = "https://rxnav.nlm.nih.gov/REST/rxcui.json"+REQUEST_TIMEOUT = 10+++def validate_drug_name(drug_name: str) -> tuple[bool, str | None]:+ """+ Query RxNorm API to validate a drug name.+ Returns (is_valid, rxcui_or_none).+ """+ try:+ response = requests.get(+ RXNORM_BASE_URL,+ params={"name": drug_name},+ timeout=REQUEST_TIMEOUT,+ )+ response.raise_for_status()+ data = response.json()++ id_group = data.get("idGroup", {})+ rxnorm_ids = id_group.get("rxnormId", [])++ if rxnorm_ids:+ return True, rxnorm_ids[0]+ return False, None++ except requests.exceptions.Timeout:+ return False, None+ except requests.exceptions.RequestException:+ return False, None+++def validate_prescription_drugs(prescription: Prescription) -> Prescription:+ """Validate all drug names in a prescription against RxNorm and update in-place."""+ for med in prescription.medications:+ if not med.drug_name or med.drug_name.strip() == "":+ med.is_validated = False+ med.validation_note = "Drug name is empty or illegible"+ continue++ is_valid, rxcui = validate_drug_name(med.drug_name.strip())++ if is_valid:+ med.is_validated = True+ med.validation_note = f"Validated — RxNorm ID: {rxcui}"+ else:+ med.is_validated = False+ med.validation_note = "Drug name not found in RxNorm — possible misread"++ return prescription