A curated collection of practical AI projects implementing OCR systems, RAG, AI agents, and other AI use cases.
feat: add Multimodal Weather App using Mistral vision and tool calling
Tiioluwani9cc286d7c11f
diff --git a/multimodal/multimodal_weather_app/.env.example b/multimodal/multimodal_weather_app/.env.examplenew file mode 100644index 0000000..b263636--- /dev/null+++ b/multimodal/multimodal_weather_app/.env.example@@ -0,0 +1 @@+MISTRAL_API_KEY=your_mistral_api_key_herediff --git a/multimodal/multimodal_weather_app/README.md b/multimodal/multimodal_weather_app/README.mdnew file mode 100644index 0000000..ee9c45d--- /dev/null+++ b/multimodal/multimodal_weather_app/README.md@@ -0,0 +1,106 @@+# Multimodal Weather App++> Upload a map image and get live weather — Mistral Small identifies the city via vision, then fetches real-time conditions through native tool calling.++## Overview++This app combines two AI capabilities in a single pipeline: **vision** and **tool calling**. You upload any map image, Mistral Small 4 reads it to identify the city, and then the same model calls a weather tool to retrieve live conditions — all displayed in a clean Gradio interface.++## Demo++++## Features++- **Vision-powered city detection** — Mistral Small 4 reads map images and extracts the city name+- **Native tool calling** — Mistral Small 4 decides when and how to invoke the weather tool+- **Live weather data** — temperature, conditions, humidity, and wind speed fetched in real time+- **No weather API key needed** — uses `python_weather`, a free weather client+- **Simple one-click UI** — built with Gradio Blocks++## Tech Stack++| Layer | Technology |+|---|---|+| LLM (vision + tool calling) | Mistral Small 4 (`mistral-small-latest`) via Mistral AI API |+| Weather data | `python_weather` (free, no API key required) |+| LLM client | `langchain-mistralai` |+| UI | Gradio |++## Prerequisites++- Python 3.10 or higher+- A Mistral AI API key — get one at [platform.mistral.ai](https://platform.mistral.ai)++## Installation++**1. Clone the repository**++```bash+git clone https://github.com/Sumanth077/Hands-On-AI-Engineering.git+cd Hands-On-AI-Engineering/multimodal/multimodal_weather_app+```++**2. Create and activate a virtual environment**++Windows:+```bash+python -m venv .venv+.venv\Scripts\activate+```++macOS/Linux:+```bash+python -m venv .venv+source .venv/bin/activate+```++**3. Install dependencies**++```bash+pip install -r requirements.txt+```++**4. Configure environment variables**++```bash+cp .env.example .env+```++Open `.env` and add your Mistral API key (see [Environment Variables](#environment-variables)).++## Usage++```bash+python app.py+```++Gradio will print a local URL (typically `http://127.0.0.1:7860`). Open it in your browser, upload a map image, and click **Identify City & Get Weather**.++**Example**++| Input | Output |+|---|---|+| Map image of Tokyo | **City:** Tokyo<br>**Temperature:** 18°C<br>**Conditions:** Partly Cloudy<br>**Humidity:** 62%<br>**Wind Speed:** 14 km/h |++## Environment Variables++Create a `.env` file in the project root with the following:++```env+MISTRAL_API_KEY=your_mistral_api_key_here+```++| Variable | Required | Description |+|---|---|---|+| `MISTRAL_API_KEY` | Yes | Your Mistral AI API key — get one at [platform.mistral.ai](https://platform.mistral.ai) |++## Project Structure++```text+multimodal-weather-app/+├── app.py # Main application — vision pipeline, tool calling, Gradio UI+├── requirements.txt # Python dependencies+├── .env.example # Environment variable template+└── .env # Your local secrets (not committed)+```diff --git a/multimodal/multimodal_weather_app/app.py b/multimodal/multimodal_weather_app/app.pynew file mode 100644index 0000000..c261230--- /dev/null+++ b/multimodal/multimodal_weather_app/app.py@@ -0,0 +1,218 @@+import os+import base64+import asyncio+import tempfile+import threading+import gradio as gr+from PIL import Image+import python_weather+from dotenv import load_dotenv+from langchain_mistralai import ChatMistralAI+from langchain_core.messages import HumanMessage++load_dotenv()++MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY")+++# ── Image helpers ────────────────────────────────────────────────────────────++def image_to_base64(image: Image.Image) -> str:+ # Write to a real temp file first to ensure a complete, valid JPEG on disk+ with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp:+ tmp_path = tmp.name+ image.convert("RGB").save(tmp_path, format="JPEG")+ with open(tmp_path, "rb") as f:+ b64 = base64.b64encode(f.read()).decode("utf-8")+ os.remove(tmp_path)+ print(f"[base64 preview]: {b64[:100]!r} (total length: {len(b64)})")+ return b64+++# ── City identification via Mistral Small (vision) ───────────────────────────++def identify_city(image: Image.Image) -> str:+ b64 = image_to_base64(image)+ llm = ChatMistralAI(model="mistral-small-latest", api_key=MISTRAL_API_KEY)+ message = HumanMessage(content=[+ {"type": "text", "text": "What city is shown in this map? Reply with the city name only."},+ {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}},+ ])+ response = llm.invoke([message])+ raw = response.content.strip()+ print(f"[mistral vision identified city]: {raw!r}")+ return raw+++# ── Async helper ─────────────────────────────────────────────────────────────++def run_async(coro):+ """Run an async coroutine in a dedicated thread with its own event loop."""+ result_holder: list = [None]+ error_holder: list = [None]++ def target():+ loop = asyncio.new_event_loop()+ asyncio.set_event_loop(loop)+ try:+ result_holder[0] = loop.run_until_complete(coro)+ except Exception as exc:+ error_holder[0] = exc+ finally:+ loop.close()++ t = threading.Thread(target=target, daemon=True)+ t.start()+ t.join()++ if error_holder[0]:+ raise error_holder[0]+ return result_holder[0]+++# ── Weather fetcher ───────────────────────────────────────────────────────────++def get_weather(city: str) -> str:+ """Fetch current weather conditions for a city. Input is the city name."""++ async def _fetch():+ async with python_weather.Client(unit=python_weather.METRIC) as client:+ w = await client.get(city)+ kind = w.kind+ if hasattr(kind, "name"):+ description = kind.name.replace("_", " ").title()+ else:+ description = str(kind).replace("WeatherKind.", "").replace("_", " ").title()+ return {+ "temperature": w.temperature,+ "description": description,+ "humidity": w.humidity,+ "wind_speed": w.wind_speed,+ }++ try:+ data = run_async(_fetch())+ return (+ f"City: {city}\n"+ f"Temperature: {data['temperature']}°C\n"+ f"Conditions: {data['description']}\n"+ f"Humidity: {data['humidity']}%\n"+ f"Wind Speed: {data['wind_speed']} km/h"+ )+ except Exception as exc:+ return f"Weather unavailable for '{city}': {exc}"+++# ── Weather tool schema + Mistral tool-calling ───────────────────────────────++_WEATHER_TOOL_SCHEMA = {+ "type": "function",+ "function": {+ "name": "get_weather",+ "description": "Fetch current weather conditions for a city.",+ "parameters": {+ "type": "object",+ "properties": {+ "city": {"type": "string", "description": "City name to get weather for."},+ },+ "required": ["city"],+ },+ },+}+++def call_weather_tool(city: str) -> str:+ llm = ChatMistralAI(model="mistral-small-latest", api_key=MISTRAL_API_KEY)+ llm_with_tools = llm.bind_tools([_WEATHER_TOOL_SCHEMA])+ response = llm_with_tools.invoke([HumanMessage(content=f"Get the current weather for {city}.")])+ for tc in response.tool_calls:+ if tc["name"] == "get_weather":+ return get_weather(tc["args"].get("city", city))+ return response.content+++# ── Output parser ─────────────────────────────────────────────────────────────++def parse_weather(text: str) -> tuple:+ temp = conditions = humidity = wind = ""+ for line in text.splitlines():+ if "Temperature:" in line:+ temp = line.split("Temperature:", 1)[1].strip()+ elif "Conditions:" in line:+ conditions = line.split("Conditions:", 1)[1].strip()+ elif "Humidity:" in line:+ humidity = line.split("Humidity:", 1)[1].strip()+ elif "Wind Speed:" in line:+ wind = line.split("Wind Speed:", 1)[1].strip()+ return temp, conditions, humidity, wind+++# ── Main pipeline ─────────────────────────────────────────────────────────────++def analyze(image):+ if image is None:+ return "Please upload a map image.", "", "", "", ""++ try:+ # Step 1: Vision — identify city from map image via Mistral Small+ city = identify_city(image)++ if not city:+ return "Mistral returned an empty response. Try a different image.", "", "", "", ""++ # Step 2: Tool calling — Mistral Small decides to call get_weather+ output = call_weather_tool(city)++ temp, conditions, humidity, wind = parse_weather(output)+ return city, temp, conditions, humidity, wind++ except Exception as exc:+ return f"Error: {exc}", "", "", "", ""+++# ── Gradio UI ─────────────────────────────────────────────────────────────────++css = """+.result-box textarea {+ font-size: 1.1rem !important;+ font-weight: 600 !important;+}+"""++with gr.Blocks(title="Multimodal Weather App", theme=gr.themes.Soft(), css=css) as demo:+ gr.Markdown(+ """+ # Multimodal Weather App+ Upload any map image. Mistral Small identifies the city via vision, then uses+ tool calling to fetch live weather using python_weather.+ """+ )++ with gr.Row():+ with gr.Column(scale=1):+ img_input = gr.Image(type="pil", label="Map Image")+ analyze_btn = gr.Button("Identify City & Get Weather", variant="primary", size="lg")++ with gr.Column(scale=1):+ gr.Markdown("### Weather Report")+ city_out = gr.Textbox(label="Identified City", interactive=False, elem_classes="result-box")+ with gr.Row():+ temp_out = gr.Textbox(label="Temperature", interactive=False)+ cond_out = gr.Textbox(label="Conditions", interactive=False)+ with gr.Row():+ hum_out = gr.Textbox(label="Humidity", interactive=False)+ wind_out = gr.Textbox(label="Wind Speed", interactive=False)++ analyze_btn.click(+ fn=analyze,+ inputs=[img_input],+ outputs=[city_out, temp_out, cond_out, hum_out, wind_out],+ )++ gr.Markdown(+ "_Powered by Mistral Small (vision + tool calling) + python_weather_",+ elem_id="footer",+ )++if __name__ == "__main__":+ demo.launch()diff --git a/multimodal/multimodal_weather_app/assets/demo.png b/multimodal/multimodal_weather_app/assets/demo.pngnew file mode 100644index 0000000..61cedfbBinary files /dev/null and b/multimodal/multimodal_weather_app/assets/demo.png differdiff --git a/multimodal/multimodal_weather_app/requirements.txt b/multimodal/multimodal_weather_app/requirements.txtnew file mode 100644index 0000000..38f3ff7--- /dev/null+++ b/multimodal/multimodal_weather_app/requirements.txt@@ -0,0 +1,6 @@+gradio>=4.44.0+python-weather>=2.0.0+Pillow>=10.0.0+python-dotenv>=1.0.0+langchain-mistralai>=0.2.0+langchain-core>=0.3.0