Skip to content

A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.

feat: add Multimodal Weather App using Mistral vision and tool calling

Tiioluwani9cc286d7c11f
diff --git a/multimodal/multimodal_weather_app/.env.example b/multimodal/multimodal_weather_app/.env.example
new file mode 100644
index 0000000..b263636
--- /dev/null
+++ b/multimodal/multimodal_weather_app/.env.example
@@ -0,0 +1 @@
+MISTRAL_API_KEY=your_mistral_api_key_here
diff --git a/multimodal/multimodal_weather_app/README.md b/multimodal/multimodal_weather_app/README.md
new file mode 100644
index 0000000..ee9c45d
--- /dev/null
+++ b/multimodal/multimodal_weather_app/README.md
@@ -0,0 +1,106 @@
+# Multimodal Weather App
+
+> Upload a map image and get live weather — Mistral Small identifies the city via vision, then fetches real-time conditions through native tool calling.
+
+## Overview
+
+This app combines two AI capabilities in a single pipeline: **vision** and **tool calling**. You upload any map image, Mistral Small 4 reads it to identify the city, and then the same model calls a weather tool to retrieve live conditions — all displayed in a clean Gradio interface.
+
+## Demo
+
+![Demo](assets/demo.png)
+
+## Features
+
+- **Vision-powered city detection** — Mistral Small 4 reads map images and extracts the city name
+- **Native tool calling** — Mistral Small 4 decides when and how to invoke the weather tool
+- **Live weather data** — temperature, conditions, humidity, and wind speed fetched in real time
+- **No weather API key needed** — uses `python_weather`, a free weather client
+- **Simple one-click UI** — built with Gradio Blocks
+
+## Tech Stack
+
+| Layer | Technology |
+|---|---|
+| LLM (vision + tool calling) | Mistral Small 4 (`mistral-small-latest`) via Mistral AI API |
+| Weather data | `python_weather` (free, no API key required) |
+| LLM client | `langchain-mistralai` |
+| UI | Gradio |
+
+## Prerequisites
+
+- Python 3.10 or higher
+- A Mistral AI API key — get one at [platform.mistral.ai](https://platform.mistral.ai)
+
+## Installation
+
+**1. Clone the repository**
+
+```bash
+git clone https://github.com/Sumanth077/Hands-On-AI-Engineering.git
+cd Hands-On-AI-Engineering/multimodal/multimodal_weather_app
+```
+
+**2. Create and activate a virtual environment**
+
+Windows:
+```bash
+python -m venv .venv
+.venv\Scripts\activate
+```
+
+macOS/Linux:
+```bash
+python -m venv .venv
+source .venv/bin/activate
+```
+
+**3. Install dependencies**
+
+```bash
+pip install -r requirements.txt
+```
+
+**4. Configure environment variables**
+
+```bash
+cp .env.example .env
+```
+
+Open `.env` and add your Mistral API key (see [Environment Variables](#environment-variables)).
+
+## Usage
+
+```bash
+python app.py
+```
+
+Gradio will print a local URL (typically `http://127.0.0.1:7860`). Open it in your browser, upload a map image, and click **Identify City & Get Weather**.
+
+**Example**
+
+| Input | Output |
+|---|---|
+| Map image of Tokyo | **City:** Tokyo<br>**Temperature:** 18°C<br>**Conditions:** Partly Cloudy<br>**Humidity:** 62%<br>**Wind Speed:** 14 km/h |
+
+## Environment Variables
+
+Create a `.env` file in the project root with the following:
+
+```env
+MISTRAL_API_KEY=your_mistral_api_key_here
+```
+
+| Variable | Required | Description |
+|---|---|---|
+| `MISTRAL_API_KEY` | Yes | Your Mistral AI API key — get one at [platform.mistral.ai](https://platform.mistral.ai) |
+
+## Project Structure
+
+```text
+multimodal-weather-app/
+├── app.py # Main application — vision pipeline, tool calling, Gradio UI
+├── requirements.txt # Python dependencies
+├── .env.example # Environment variable template
+└── .env # Your local secrets (not committed)
+```
diff --git a/multimodal/multimodal_weather_app/app.py b/multimodal/multimodal_weather_app/app.py
new file mode 100644
index 0000000..c261230
--- /dev/null
+++ b/multimodal/multimodal_weather_app/app.py
@@ -0,0 +1,218 @@
+import os
+import base64
+import asyncio
+import tempfile
+import threading
+import gradio as gr
+from PIL import Image
+import python_weather
+from dotenv import load_dotenv
+from langchain_mistralai import ChatMistralAI
+from langchain_core.messages import HumanMessage
+
+load_dotenv()
+
+MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY")
+
+
+# ── Image helpers ────────────────────────────────────────────────────────────
+
+def image_to_base64(image: Image.Image) -> str:
+ # Write to a real temp file first to ensure a complete, valid JPEG on disk
+ with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp:
+ tmp_path = tmp.name
+ image.convert("RGB").save(tmp_path, format="JPEG")
+ with open(tmp_path, "rb") as f:
+ b64 = base64.b64encode(f.read()).decode("utf-8")
+ os.remove(tmp_path)
+ print(f"[base64 preview]: {b64[:100]!r} (total length: {len(b64)})")
+ return b64
+
+
+# ── City identification via Mistral Small (vision) ───────────────────────────
+
+def identify_city(image: Image.Image) -> str:
+ b64 = image_to_base64(image)
+ llm = ChatMistralAI(model="mistral-small-latest", api_key=MISTRAL_API_KEY)
+ message = HumanMessage(content=[
+ {"type": "text", "text": "What city is shown in this map? Reply with the city name only."},
+ {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}},
+ ])
+ response = llm.invoke([message])
+ raw = response.content.strip()
+ print(f"[mistral vision identified city]: {raw!r}")
+ return raw
+
+
+# ── Async helper ─────────────────────────────────────────────────────────────
+
+def run_async(coro):
+ """Run an async coroutine in a dedicated thread with its own event loop."""
+ result_holder: list = [None]
+ error_holder: list = [None]
+
+ def target():
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ try:
+ result_holder[0] = loop.run_until_complete(coro)
+ except Exception as exc:
+ error_holder[0] = exc
+ finally:
+ loop.close()
+
+ t = threading.Thread(target=target, daemon=True)
+ t.start()
+ t.join()
+
+ if error_holder[0]:
+ raise error_holder[0]
+ return result_holder[0]
+
+
+# ── Weather fetcher ───────────────────────────────────────────────────────────
+
+def get_weather(city: str) -> str:
+ """Fetch current weather conditions for a city. Input is the city name."""
+
+ async def _fetch():
+ async with python_weather.Client(unit=python_weather.METRIC) as client:
+ w = await client.get(city)
+ kind = w.kind
+ if hasattr(kind, "name"):
+ description = kind.name.replace("_", " ").title()
+ else:
+ description = str(kind).replace("WeatherKind.", "").replace("_", " ").title()
+ return {
+ "temperature": w.temperature,
+ "description": description,
+ "humidity": w.humidity,
+ "wind_speed": w.wind_speed,
+ }
+
+ try:
+ data = run_async(_fetch())
+ return (
+ f"City: {city}\n"
+ f"Temperature: {data['temperature']}°C\n"
+ f"Conditions: {data['description']}\n"
+ f"Humidity: {data['humidity']}%\n"
+ f"Wind Speed: {data['wind_speed']} km/h"
+ )
+ except Exception as exc:
+ return f"Weather unavailable for '{city}': {exc}"
+
+
+# ── Weather tool schema + Mistral tool-calling ───────────────────────────────
+
+_WEATHER_TOOL_SCHEMA = {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Fetch current weather conditions for a city.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "city": {"type": "string", "description": "City name to get weather for."},
+ },
+ "required": ["city"],
+ },
+ },
+}
+
+
+def call_weather_tool(city: str) -> str:
+ llm = ChatMistralAI(model="mistral-small-latest", api_key=MISTRAL_API_KEY)
+ llm_with_tools = llm.bind_tools([_WEATHER_TOOL_SCHEMA])
+ response = llm_with_tools.invoke([HumanMessage(content=f"Get the current weather for {city}.")])
+ for tc in response.tool_calls:
+ if tc["name"] == "get_weather":
+ return get_weather(tc["args"].get("city", city))
+ return response.content
+
+
+# ── Output parser ─────────────────────────────────────────────────────────────
+
+def parse_weather(text: str) -> tuple:
+ temp = conditions = humidity = wind = ""
+ for line in text.splitlines():
+ if "Temperature:" in line:
+ temp = line.split("Temperature:", 1)[1].strip()
+ elif "Conditions:" in line:
+ conditions = line.split("Conditions:", 1)[1].strip()
+ elif "Humidity:" in line:
+ humidity = line.split("Humidity:", 1)[1].strip()
+ elif "Wind Speed:" in line:
+ wind = line.split("Wind Speed:", 1)[1].strip()
+ return temp, conditions, humidity, wind
+
+
+# ── Main pipeline ─────────────────────────────────────────────────────────────
+
+def analyze(image):
+ if image is None:
+ return "Please upload a map image.", "", "", "", ""
+
+ try:
+ # Step 1: Vision — identify city from map image via Mistral Small
+ city = identify_city(image)
+
+ if not city:
+ return "Mistral returned an empty response. Try a different image.", "", "", "", ""
+
+ # Step 2: Tool calling — Mistral Small decides to call get_weather
+ output = call_weather_tool(city)
+
+ temp, conditions, humidity, wind = parse_weather(output)
+ return city, temp, conditions, humidity, wind
+
+ except Exception as exc:
+ return f"Error: {exc}", "", "", "", ""
+
+
+# ── Gradio UI ─────────────────────────────────────────────────────────────────
+
+css = """
+.result-box textarea {
+ font-size: 1.1rem !important;
+ font-weight: 600 !important;
+}
+"""
+
+with gr.Blocks(title="Multimodal Weather App", theme=gr.themes.Soft(), css=css) as demo:
+ gr.Markdown(
+ """
+ # Multimodal Weather App
+ Upload any map image. Mistral Small identifies the city via vision, then uses
+ tool calling to fetch live weather using python_weather.
+ """
+ )
+
+ with gr.Row():
+ with gr.Column(scale=1):
+ img_input = gr.Image(type="pil", label="Map Image")
+ analyze_btn = gr.Button("Identify City & Get Weather", variant="primary", size="lg")
+
+ with gr.Column(scale=1):
+ gr.Markdown("### Weather Report")
+ city_out = gr.Textbox(label="Identified City", interactive=False, elem_classes="result-box")
+ with gr.Row():
+ temp_out = gr.Textbox(label="Temperature", interactive=False)
+ cond_out = gr.Textbox(label="Conditions", interactive=False)
+ with gr.Row():
+ hum_out = gr.Textbox(label="Humidity", interactive=False)
+ wind_out = gr.Textbox(label="Wind Speed", interactive=False)
+
+ analyze_btn.click(
+ fn=analyze,
+ inputs=[img_input],
+ outputs=[city_out, temp_out, cond_out, hum_out, wind_out],
+ )
+
+ gr.Markdown(
+ "_Powered by Mistral Small (vision + tool calling) + python_weather_",
+ elem_id="footer",
+ )
+
+if __name__ == "__main__":
+ demo.launch()
diff --git a/multimodal/multimodal_weather_app/assets/demo.png b/multimodal/multimodal_weather_app/assets/demo.png
new file mode 100644
index 0000000..61cedfb
Binary files /dev/null and b/multimodal/multimodal_weather_app/assets/demo.png differ
diff --git a/multimodal/multimodal_weather_app/requirements.txt b/multimodal/multimodal_weather_app/requirements.txt
new file mode 100644
index 0000000..38f3ff7
--- /dev/null
+++ b/multimodal/multimodal_weather_app/requirements.txt
@@ -0,0 +1,6 @@
+gradio>=4.44.0
+python-weather>=2.0.0
+Pillow>=10.0.0
+python-dotenv>=1.0.0
+langchain-mistralai>=0.2.0
+langchain-core>=0.3.0