Personal AI, On Personal Devices
Python
feat(connectors): generic IMAP email connector for any provider (#518) (#586)
Aditya Malikb266f16950d3
diff --git a/frontend/src/components/setup/SourceConnectFlow.tsx b/frontend/src/components/setup/SourceConnectFlow.tsxindex d33e9f6e..cc55fce5 100644--- a/frontend/src/components/setup/SourceConnectFlow.tsx+++ b/frontend/src/components/setup/SourceConnectFlow.tsx@@ -283,9 +283,11 @@ function StepByStepPanel({onSkip: () => void;isConnecting: boolean;}) {- const [inputs, setInputs] = useState<Record<string, string>>({});const steps = connector.steps || [];const fields = connector.inputFields || [];+ const [inputs, setInputs] = useState<Record<string, string>>(() =>+ Object.fromEntries(fields.map((field) => [field.name, field.defaultValue || ''])),+ );const updateInput = (name: string, value: string) => {setInputs((prev) => ({ ...prev, [name]: value }));@@ -298,6 +300,11 @@ function StepByStepPanel({else if (field.name === 'password') req.password = inputs.password;else if (field.name === 'token') req.token = inputs.token;else if (field.name === 'path') req.path = inputs.path;+ else if (field.name === 'host' && inputs.host?.trim()) req.host = inputs.host.trim();+ else if (field.name === 'port' && inputs.port?.trim()) req.port = Number(inputs.port);+ else if (field.name === 'security' && (inputs.security === 'tls' || inputs.security === 'starttls')) {+ req.security = inputs.security;+ }}// For email+password connectors, also set token as email:passwordif (req.email && req.password) {@@ -310,7 +317,9 @@ function StepByStepPanel({onConnect(req);};- const allFilled = fields.every((f) => inputs[f.name]?.trim());+ const allFilled = fields+ .filter((field) => field.required !== false)+ .every((field) => inputs[field.name]?.trim());return (<div style={{ padding: '0 4px' }}>@@ -378,24 +387,49 @@ function StepByStepPanel({marginBottom: 10,}}>{fields.map((field) => (- <input- key={field.name}- value={inputs[field.name] || ''}- onChange={(e) => updateInput(field.name, e.target.value)}- placeholder={field.placeholder}- type={field.type || 'text'}- style={{- width: '100%',- padding: '8px 10px',- background: 'var(--color-bg-secondary)',- border: '1px solid var(--color-border)',- borderRadius: 4,- color: 'var(--color-text)',- fontSize: 13,- marginBottom: 8,- boxSizing: 'border-box',- }}- />+ field.type === 'select' ? (+ <select+ key={field.name}+ value={inputs[field.name] || field.defaultValue || ''}+ onChange={(e) => updateInput(field.name, e.target.value)}+ aria-label={field.placeholder}+ style={{+ width: '100%',+ padding: '8px 10px',+ background: 'var(--color-bg-secondary)',+ border: '1px solid var(--color-border)',+ borderRadius: 4,+ color: 'var(--color-text)',+ fontSize: 13,+ marginBottom: 8,+ boxSizing: 'border-box',+ }}+ >+ {(field.options || []).map((option) => (+ <option key={option.value} value={option.value}>{option.label}</option>+ ))}+ </select>+ ) : (+ <input+ key={field.name}+ value={inputs[field.name] || ''}+ onChange={(e) => updateInput(field.name, e.target.value)}+ placeholder={field.placeholder}+ type={field.type || 'text'}+ required={field.required !== false}+ style={{+ width: '100%',+ padding: '8px 10px',+ background: 'var(--color-bg-secondary)',+ border: '1px solid var(--color-border)',+ borderRadius: 4,+ color: 'var(--color-text)',+ fontSize: 13,+ marginBottom: 8,+ boxSizing: 'border-box',+ }}+ />+ )))}</div>)}@@ -549,6 +583,7 @@ export function SourceConnectFlow({</div>) : activeCard.steps ? (<StepByStepPanel+ key={activeCard.connector_id}connector={activeCard}onConnect={(req) => handleConnect(activeEntry.id, req)}onSkip={() => handleSkip(activeEntry.id)}diff --git a/frontend/src/pages/DataSourcesPage.tsx b/frontend/src/pages/DataSourcesPage.tsxindex 795acf02..0ed27349 100644--- a/frontend/src/pages/DataSourcesPage.tsx+++ b/frontend/src/pages/DataSourcesPage.tsx@@ -23,7 +23,7 @@ import {} from 'lucide-react';import type { LucideIcon } from 'lucide-react';import { SOURCE_CATALOG } from '../types/connectors';-import type { ConnectRequest } from '../types/connectors';+import type { ConnectRequest, ConnectorMeta } from '../types/connectors';import { listConnectors, connectSource, disconnectSource, getSyncStatus, triggerSync, startServerOAuth } from '../lib/connectors-api';import type { SyncStatus } from '../types/connectors';@@ -36,16 +36,20 @@ function InlineConnectForm({loading,onSubmit,}: {- fields: Array<{ name: string; placeholder: string; type?: string }>;+ fields: NonNullable<ConnectorMeta['inputFields']>;loading: boolean;onSubmit: (req: ConnectRequest) => void;}) {- const [inputs, setInputs] = useState<Record<string, string>>({});+ const [inputs, setInputs] = useState<Record<string, string>>(() =>+ Object.fromEntries(fields.map((field) => [field.name, field.defaultValue || ''])),+ );const update = (name: string, value: string) =>setInputs((p) => ({ ...p, [name]: value }));- const allFilled = fields.every((f) => inputs[f.name]?.trim());+ const allFilled = fields+ .filter((field) => field.required !== false)+ .every((field) => inputs[field.name]?.trim());const submit = () => {const req: ConnectRequest = {};@@ -54,6 +58,11 @@ function InlineConnectForm({else if (f.name === 'password') req.password = inputs.password;else if (f.name === 'token') req.token = inputs.token;else if (f.name === 'path') req.path = inputs.path;+ else if (f.name === 'host' && inputs.host?.trim()) req.host = inputs.host.trim();+ else if (f.name === 'port' && inputs.port?.trim()) req.port = Number(inputs.port);+ else if (f.name === 'security' && (inputs.security === 'tls' || inputs.security === 'starttls')) {+ req.security = inputs.security;+ }}if (req.email && req.password) {req.token = `${req.email}:${req.password}`;@@ -66,21 +75,43 @@ function InlineConnectForm({return (<div>{fields.map((f) => (- <input- key={f.name}- value={inputs[f.name] || ''}- onChange={(e) => update(f.name, e.target.value)}- placeholder={f.placeholder}- type={f.type || 'text'}- style={{- width: '100%', padding: '7px 10px',- background: 'var(--color-bg)',- border: '1px solid var(--color-border)',- borderRadius: 4, color: 'var(--color-text)',- fontSize: 12, marginBottom: 6,- boxSizing: 'border-box',- }}- />+ f.type === 'select' ? (+ <select+ key={f.name}+ value={inputs[f.name] || f.defaultValue || ''}+ onChange={(e) => update(f.name, e.target.value)}+ aria-label={f.placeholder}+ style={{+ width: '100%', padding: '7px 10px',+ background: 'var(--color-bg)',+ border: '1px solid var(--color-border)',+ borderRadius: 4, color: 'var(--color-text)',+ fontSize: 12, marginBottom: 6,+ boxSizing: 'border-box',+ }}+ >+ {(f.options || []).map((option) => (+ <option key={option.value} value={option.value}>{option.label}</option>+ ))}+ </select>+ ) : (+ <input+ key={f.name}+ value={inputs[f.name] || ''}+ onChange={(e) => update(f.name, e.target.value)}+ placeholder={f.placeholder}+ type={f.type || 'text'}+ required={f.required !== false}+ style={{+ width: '100%', padding: '7px 10px',+ background: 'var(--color-bg)',+ border: '1px solid var(--color-border)',+ borderRadius: 4, color: 'var(--color-text)',+ fontSize: 12, marginBottom: 6,+ boxSizing: 'border-box',+ }}+ />+ )))}<buttononClick={submit}diff --git a/frontend/src/types/connectors.ts b/frontend/src/types/connectors.tsindex 76091396..146f173f 100644--- a/frontend/src/types/connectors.ts+++ b/frontend/src/types/connectors.ts@@ -18,7 +18,10 @@ export interface ConnectorMeta {inputFields?: Array<{name: string;placeholder: string;- type?: 'text' | 'password';+ type?: 'text' | 'password' | 'number' | 'select';+ required?: boolean;+ defaultValue?: string;+ options?: Array<{ value: string; label: string }>;}>;}@@ -53,6 +56,9 @@ export interface ConnectRequest {code?: string;email?: string;password?: string;+ host?: string;+ port?: number;+ security?: 'tls' | 'starttls';}/** Response from POST /v1/connectors/{id}/connect.@@ -120,6 +126,52 @@ export const SOURCE_CATALOG: ConnectorMeta[] = [{ name: 'password', placeholder: 'App password (xxxx xxxx xxxx xxxx)', type: 'password' },],},+ {+ connector_id: 'imap',+ display_name: 'Email (IMAP)',+ auth_type: 'oauth',+ category: 'communication',+ icon: 'Mail',+ color: 'text-amber-400',+ description: 'Any IMAP mailbox',+ unitLabel: 'emails',+ steps: [+ {+ label: 'Generate an app password with your email provider (IMAP must be enabled), then enter your email address and app password. Known providers are detected automatically; use the optional server settings for other providers.',+ },+ ],+ troubleshooting: [+ 'Most providers require an app password rather than your normal password.',+ 'Use TLS for implicit TLS (usually port 993), or STARTTLS (usually port 143).',+ 'For security, private, loopback, link-local, and otherwise non-public server addresses are rejected.',+ ],+ inputFields: [+ { name: 'email', placeholder: '[email protected]', type: 'text' },+ { name: 'password', placeholder: 'App password', type: 'password' },+ {+ name: 'host',+ placeholder: 'IMAP host (optional; auto-detected)',+ type: 'text',+ required: false,+ },+ {+ name: 'port',+ placeholder: 'Port (optional; 993 or 143)',+ type: 'number',+ required: false,+ },+ {+ name: 'security',+ placeholder: 'Connection security',+ type: 'select',+ defaultValue: 'tls',+ options: [+ { value: 'tls', label: 'TLS (usually port 993)' },+ { value: 'starttls', label: 'STARTTLS (usually port 143)' },+ ],+ },+ ],+ },{connector_id: 'slack',display_name: 'Slack',diff --git a/src/openjarvis/connectors/__init__.py b/src/openjarvis/connectors/__init__.pyindex 74497b3e..3ff9d83a 100644--- a/src/openjarvis/connectors/__init__.py+++ b/src/openjarvis/connectors/__init__.py@@ -78,6 +78,11 @@ try:except ImportError:pass+try:+ import openjarvis.connectors.imap # noqa: F401+except ImportError:+ pass+try:import openjarvis.connectors.gcalendar # noqa: F401except ImportError:diff --git a/src/openjarvis/connectors/gmail_imap.py b/src/openjarvis/connectors/gmail_imap.pyindex 61836bae..1d00d2a2 100644--- a/src/openjarvis/connectors/gmail_imap.py+++ b/src/openjarvis/connectors/gmail_imap.py@@ -11,9 +11,11 @@ from __future__ import annotationsimport email as email_libimport imaplibimport logging+import sslfrom datetime import datetimefrom email.header import decode_headerfrom email.utils import parsedate_to_datetime+from imaplib import IMAP4from typing import Iterator, List, Optionalfrom openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus@@ -25,19 +27,57 @@ from openjarvis.tools._stubs import ToolSpeclogger = logging.getLogger(__name__)_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "gmail_imap.json")+_IMAP_TIMEOUT_SECONDS = 30+_SECURITY_TLS = "tls"+_SECURITY_STARTTLS = "starttls"+_UIDVALIDITY_ATTRIBUTE = "_openjarvis_uidvalidity"+++def _normalize_imap_security(value: str) -> str:+ """Normalize an explicit encrypted IMAP transport mode."""+ normalized = (value or _SECURITY_TLS).strip().lower().replace("-", "")+ if normalized in {"tls", "ssl", "implicit", "implicittls"}:+ return _SECURITY_TLS+ if normalized == "starttls":+ return _SECURITY_STARTTLS+ raise ValueError("IMAP security must be 'tls' or 'starttls'")+++def _normalize_imap_port(value: object, security: str) -> int:+ """Return a valid port, defaulting from the selected transport mode."""+ if value is None or value == "":+ return 993 if security == _SECURITY_TLS else 143+ if isinstance(value, bool):+ raise ValueError("IMAP port must be an integer between 1 and 65535")+ try:+ port = int(value)+ except (TypeError, ValueError) as exc:+ raise ValueError("IMAP port must be an integer between 1 and 65535") from exc+ if not 1 <= port <= 65535:+ raise ValueError("IMAP port must be an integer between 1 and 65535")+ return port++def _decode_header(raw: object) -> str:+ """Decode an email header (str or Header) to a safe plain string.-def _decode_subject(raw: str) -> str:- """Decode a possibly-encoded email subject header."""+ Real mail can carry raw 8-bit bytes in headers, which makes ``msg.get()``+ return an ``email.header.Header`` (not JSON serializable) and yields bogus+ charsets like ``unknown-8bit``. Coerce to a string, replacing what can't be+ decoded.+ """if not raw:return ""- decoded_parts = decode_header(raw)- return "".join(- part.decode(enc or "utf-8", errors="replace")- if isinstance(part, bytes)- else part- for part, enc in decoded_parts- )+ out = []+ for part, enc in decode_header(raw if isinstance(raw, str) else str(raw)):+ if isinstance(part, bytes):+ try:+ out.append(part.decode(enc or "utf-8", errors="replace"))+ except LookupError:+ out.append(part.decode("utf-8", errors="replace"))+ else:+ out.append(part)+ return "".join(out)def _extract_text_body(msg: email_lib.message.Message) -> str:@@ -73,6 +113,28 @@ def _parse_date(msg: email_lib.message.Message) -> datetime:return datetime.now()+def _read_uidvalidity(imap: imaplib.IMAP4) -> str:+ """Read the UID generation for the currently selected mailbox."""+ response_type, data = imap.response("UIDVALIDITY")+ if isinstance(response_type, bytes):+ response_type = response_type.decode("ascii", errors="replace")+ if str(response_type).upper() != "UIDVALIDITY" or not data:+ raise IMAP4.error("IMAP server did not provide UIDVALIDITY")++ raw_value = data[-1]+ if isinstance(raw_value, bytes):+ text = raw_value.decode("ascii", errors="strict").strip()+ else:+ text = str(raw_value).strip()+ try:+ value = int(text)+ except (TypeError, ValueError) as exc:+ raise IMAP4.error("IMAP server provided invalid UIDVALIDITY") from exc+ if value <= 0:+ raise IMAP4.error("IMAP server provided invalid UIDVALIDITY")+ return str(value)++@ConnectorRegistry.register("gmail_imap")class GmailIMAPConnector(BaseConnector):"""Gmail connector using IMAP + app password.@@ -92,12 +154,19 @@ class GmailIMAPConnector(BaseConnector):credentials_path: str = "",*,imap_host: str = "",+ imap_port: int | None = None,+ imap_security: str = "",max_messages: Optional[int] = None,) -> None:self._email = email_addressself._password = app_passwordself._credentials_path = credentials_path or _DEFAULT_CREDENTIALS_PATHself._imap_host = imap_host or self._default_imap_host+ self._imap_port = imap_port+ self._imap_security = imap_security+ # None means "use the persisted validation marker". Direct credentials+ # are not considered connected until a real LOGIN + SELECT succeeds.+ self._credentials_validated: bool | None = None# ``None`` means "no cap" — the full inbox is indexed. A positive# value is still honored for tests that want a bounded scan.self._max_messages = max_messages@@ -106,20 +175,53 @@ class GmailIMAPConnector(BaseConnector):def _resolve_credentials(self) -> tuple[str, str]:"""Return (email, password) — direct args take priority."""- if self._email and self._password:- return self._email, self._password- tokens = load_tokens(self._credentials_path)- if tokens:- return tokens.get("email", ""), tokens.get("password", "")- return "", ""+ email_address, password, _, _, _, _ = self._resolve_connection()+ return email_address, password++ def _host_for_email(self, email_address: str) -> str:+ """Return the connector's provider host for *email_address*."""+ return self._default_imap_host++ def _resolve_connection(self) -> tuple[str, str, str, int, str, bool]:+ """Resolve credentials and transport settings from args or storage."""+ tokens = load_tokens(self._credentials_path) or {}+ using_direct_credentials = bool(self._email and self._password)+ if using_direct_credentials:+ email_address, password = self._email, self._password+ stored_host = ""+ stored_security = ""+ stored_port = None+ else:+ email_address = str(tokens.get("email", ""))+ password = str(tokens.get("password", ""))+ stored_host = str(tokens.get("imap_host", ""))+ stored_security = str(tokens.get("imap_security", ""))+ stored_port = tokens.get("imap_port")++ host = (+ self._imap_host or stored_host or self._host_for_email(email_address)+ ).strip()+ security = _normalize_imap_security(self._imap_security or stored_security)+ port = _normalize_imap_port(+ self._imap_port if self._imap_port is not None else stored_port,+ security,+ )+ if self._credentials_validated is not None:+ validated = self._credentials_validated+ elif using_direct_credentials:+ validated = False+ else:+ validated = tokens.get("validated") is True+ return email_address, password, host, port, security, validateddef is_connected(self) -> bool:- em, pw = self._resolve_credentials()- return bool(em and pw)+ em, pw, host, _, _, validated = self._resolve_connection()+ return bool(em and pw and host and validated)def disconnect(self) -> None:self._email = ""self._password = ""+ self._credentials_validated = Falsedelete_tokens(self._credentials_path)def auth_url(self) -> str:@@ -127,18 +229,128 @@ class GmailIMAPConnector(BaseConnector):def handle_callback(self, code: str) -> None:# code format: "email:password"- if ":" in code:- em, pw = code.split(":", 1)- save_tokens(- self._credentials_path,- {"email": em.strip(), "password": pw.strip()},- )- else:- save_tokens(- self._credentials_path,- {"email": "", "password": code.strip()},+ if ":" not in code:+ raise ValueError("Expected credentials in 'email:password' format")+ em, pw = code.split(":", 1)+ self.connect_with_credentials(em, pw)++ def connect_with_credentials(+ self,+ email_address: str,+ app_password: str,+ *,+ imap_host: str = "",+ imap_port: int | None = None,+ imap_security: str = "",+ ) -> None:+ """Validate LOGIN + INBOX access, then persist the connection."""+ email_address = email_address.strip()+ app_password = app_password.strip()+ if not email_address or not app_password:+ raise ValueError("Email address and app password are required")++ host = (imap_host or self._host_for_email(email_address)).strip()+ if not host:+ raise ValueError("An IMAP host is required")+ security = _normalize_imap_security(imap_security)+ port = _normalize_imap_port(imap_port, security)++ imap = self._open_authenticated_imap(+ email_address,+ app_password,+ host,+ port,+ security,+ )+ self._close_imap(imap)++ save_tokens(+ self._credentials_path,+ {+ "email": email_address,+ "password": app_password,+ "imap_host": host,+ "imap_port": port,+ "imap_security": security,+ "validated": True,+ },+ )+ self._email = email_address+ self._password = app_password+ self._imap_host = host+ self._imap_port = port+ self._imap_security = security+ self._credentials_validated = True++ def _make_imap_client(self, host: str, port: int, security: str) -> imaplib.IMAP4:+ """Create a TLS or STARTTLS IMAP client."""+ tls_context = ssl.create_default_context()+ if security == _SECURITY_TLS:+ return imaplib.IMAP4_SSL(+ host,+ port,+ ssl_context=tls_context,+ timeout=_IMAP_TIMEOUT_SECONDS,)+ imap = imaplib.IMAP4(host, port, timeout=_IMAP_TIMEOUT_SECONDS)+ try:+ status, _ = imap.starttls(ssl_context=tls_context)+ if status != "OK":+ raise IMAP4.error("IMAP server rejected STARTTLS")+ return imap+ except Exception:+ self._close_imap(imap)+ raise++ def _open_authenticated_imap(+ self,+ email_address: str,+ password: str,+ host: str,+ port: int,+ security: str,+ ) -> imaplib.IMAP4:+ """Open, authenticate, and select INBOX on one IMAP connection."""+ imap = self._make_imap_client(host, port, security)+ try:+ status, _ = imap.login(email_address, password)+ if status != "OK":+ raise IMAP4.error("IMAP login rejected")+ status, _ = imap.select("INBOX", readonly=True)+ if status != "OK":+ raise IMAP4.error("Unable to select INBOX read-only")+ # A UID is meaningful only together with this mailbox generation.+ # Store it on the selected connection so reconnects cannot reuse an+ # old UID against a reset or replaced mailbox.+ setattr(imap, _UIDVALIDITY_ATTRIBUTE, _read_uidvalidity(imap))+ return imap+ except Exception:+ self._close_imap(imap)+ raise++ def _open_imap(self) -> imaplib.IMAP4:+ """Open an authenticated, INBOX-selected IMAP connection."""+ em, pw, host, port, security, _ = self._resolve_connection()+ if not em or not pw or not host:+ raise ValueError("Complete IMAP credentials are required")+ imap = self._open_authenticated_imap(em, pw, host, port, security)+ self._credentials_validated = True+ return imap++ @staticmethod+ def _close_imap(imap: imaplib.IMAP4 | None) -> None:+ """Best-effort logout for normal, failed, and cancelled syncs."""+ if imap is None:+ return+ try:+ imap.logout()+ except Exception:+ try:+ imap.shutdown()+ except Exception:+ pass+def sync(self,*,@@ -149,69 +361,124 @@ class GmailIMAPConnector(BaseConnector):if not em or not pw:return- imap = imaplib.IMAP4_SSL(self._imap_host)try:- imap.login(em, pw)- except imaplib.IMAP4.error as exc:+ imap = self._open_imap()+ except (IMAP4.error, OSError, ValueError) as exc:+ if isinstance(exc, (IMAP4.error, ValueError)):+ self._credentials_validated = Falselogger.error("IMAP login failed: %s", exc)return- imap.select("INBOX", readonly=True)-- # Always SEARCH ALL. IMAP has no native cursor that survives a- # server restart, so applying the SyncEngine's ``since`` filter- # during a partial backfill would silently skip the older- # unprocessed messages. The pipeline-level dedup (_seen_doc_ids- # set + INSERT OR IGNORE in KnowledgeStore) makes re-scanning- # already-indexed messages cheap, so resume is correct as long- # as we keep enumerating the full inbox.- _, data = imap.search(None, "ALL")- msg_ids = data[0].split()- self._items_total = len(msg_ids)-- # Newest-first iteration so Deep Research becomes useful while- # the long tail of older mail finishes indexing in the- # background. IMAP returns sequence numbers in arrival order- # (oldest -> newest), so reversing puts the most recent first.- ordered = list(reversed(msg_ids))- if self._max_messages is not None and self._max_messages > 0:- ordered = ordered[: self._max_messages]synced = 0-- for mid in ordered:- try:- _, msg_data = imap.fetch(mid, "(RFC822)")- raw = msg_data[0][1]- msg = email_lib.message_from_bytes(raw)- except Exception:- continue-- subject = _decode_subject(msg.get("Subject", ""))- sender = msg.get("From", "")- to = msg.get("To", "")- body = _extract_text_body(msg)- timestamp = _parse_date(msg)- message_id = msg.get("Message-ID", mid.decode())-- synced += 1- yield Document(- doc_id=f"gmail:{message_id}",- source="gmail",- doc_type="email",- content=body,- title=subject,- author=sender,- participants=[a.strip() for a in (to or "").split(",") if a.strip()],- timestamp=timestamp,- thread_id=msg.get("In-Reply-To", ""),- url="https://mail.google.com/mail/u/0/#inbox",- metadata={- "message_id": message_id,- },- )-- imap.logout()- self._items_synced = synced+ try:+ uidvalidity = str(getattr(imap, _UIDVALIDITY_ATTRIBUTE))+ # SEARCH ALL is deliberate: pipeline dedup makes a complete scan+ # safe while avoiding gaps after an interrupted backfill.+ status, data = imap.uid("SEARCH", None, "ALL")+ if status != "OK" or not data or data[0] is None:+ logger.error("IMAP UID SEARCH ALL failed: %s", status)+ return+ message_uids = data[0].split() if data[0] else []+ self._items_total = len(message_uids)+ ordered = list(reversed(message_uids))+ if self._max_messages is not None and self._max_messages > 0:+ ordered = ordered[: self._max_messages]++ max_reconnects = 5+ consecutive_reconnects = 0+ index = 0+ while index < len(ordered):+ message_uid = ordered[index]+ try:+ status, msg_data = imap.uid("FETCH", message_uid, "(RFC822)")+ message_record = next(+ (+ item+ for item in msg_data or []+ if isinstance(item, tuple)+ and len(item) > 1+ and isinstance(item[1], bytes)+ ),+ None,+ )+ if status != "OK" or message_record is None:+ index += 1+ continue+ raw = message_record[1]+ msg = email_lib.message_from_bytes(raw)+ except (IMAP4.abort, OSError) as exc:+ if consecutive_reconnects >= max_reconnects:+ logger.error(+ "IMAP sync stopping after %d reconnect attempts: %s",+ consecutive_reconnects,+ exc,+ )+ break+ consecutive_reconnects += 1+ logger.warning(+ "IMAP connection dropped (%s); reconnecting %d/%d",+ exc,+ consecutive_reconnects,+ max_reconnects,+ )+ self._close_imap(imap)+ imap = None+ try:+ imap = self._open_imap()+ except (IMAP4.error, OSError, ValueError) as reconnect_exc:+ logger.error("IMAP reconnect failed: %s", reconnect_exc)+ break+ reconnected_uidvalidity = str(getattr(imap, _UIDVALIDITY_ATTRIBUTE))+ if reconnected_uidvalidity != uidvalidity:+ logger.error(+ "IMAP UIDVALIDITY changed from %s to %s during sync; "+ "stopping before reusing stale UIDs",+ uidvalidity,+ reconnected_uidvalidity,+ )+ break+ continue+ except Exception:+ index += 1+ continue++ subject = _decode_header(msg.get("Subject", ""))+ sender = _decode_header(msg.get("From", ""))+ to = _decode_header(msg.get("To", ""))+ body = _extract_text_body(msg)+ timestamp = _parse_date(msg)+ uid_text = message_uid.decode("ascii", errors="replace")+ message_id = _decode_header(msg.get("Message-ID", "")) or (+ f"{uidvalidity}:{uid_text}"+ )++ synced += 1+ index += 1+ consecutive_reconnects = 0+ yield Document(+ doc_id=f"gmail:{message_id}",+ source="gmail",+ doc_type="email",+ content=body,+ title=subject,+ author=sender,+ participants=[+ address.strip()+ for address in (to or "").split(",")+ if address.strip()+ ],+ timestamp=timestamp,+ thread_id=_decode_header(msg.get("In-Reply-To", "")),+ url="https://mail.google.com/mail/u/0/#inbox",+ metadata={+ "message_id": message_id,+ "imap_uid": uid_text,+ "imap_uidvalidity": uidvalidity,+ },+ )+ finally:+ self._close_imap(imap)+ self._items_synced = synceddef sync_status(self) -> SyncStatus:return SyncStatus(diff --git a/src/openjarvis/connectors/imap.py b/src/openjarvis/connectors/imap.pynew file mode 100644index 00000000..08f43015--- /dev/null+++ b/src/openjarvis/connectors/imap.py@@ -0,0 +1,282 @@+"""Generic IMAP email connector for any provider."""++from __future__ import annotations++import imaplib+import ipaddress+import socket+import ssl+from datetime import datetime+from typing import Iterator, Optional++from openjarvis.connectors._stubs import Document+from openjarvis.connectors.gmail_imap import (+ _IMAP_TIMEOUT_SECONDS,+ _SECURITY_STARTTLS,+ _SECURITY_TLS,+ GmailIMAPConnector,+)+from openjarvis.core.config import DEFAULT_CONFIG_DIR+from openjarvis.core.registry import ConnectorRegistry++_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "imap.json")++_PROVIDER_HOSTS = {+ "gmail.com": "imap.gmail.com",+ "googlemail.com": "imap.gmail.com",+ "outlook.com": "outlook.office365.com",+ "hotmail.com": "outlook.office365.com",+ "live.com": "outlook.office365.com",+ "icloud.com": "imap.mail.me.com",+ "me.com": "imap.mail.me.com",+ "yahoo.com": "imap.mail.yahoo.com",+ "fastmail.com": "imap.fastmail.com",+ "aol.com": "imap.aol.com",+ "zoho.com": "imap.zoho.com",+ "gmx.com": "imap.gmx.com",+}++_BLOCKED_HOSTNAMES = frozenset(+ {+ "localhost",+ "metadata.google.internal",+ "metadata.google.com",+ }+)+++def _normalize_hostname(host: str) -> str:+ """Normalize and validate a host without accepting URL-like input."""+ normalized = host.strip().rstrip(".")+ if normalized.startswith("[") and normalized.endswith("]"):+ normalized = normalized[1:-1]+ if not normalized or any(char.isspace() for char in normalized):+ raise ValueError("A valid IMAP hostname is required")+ if any(char in normalized for char in ("/", "\\", "@")) or "://" in normalized:+ raise ValueError("IMAP host must be a hostname or IP address, not a URL")++ try:+ return str(ipaddress.ip_address(normalized))+ except ValueError:+ pass++ try:+ ascii_host = normalized.encode("idna").decode("ascii").lower()+ except UnicodeError as exc:+ raise ValueError("A valid IMAP hostname is required") from exc+ labels = ascii_host.split(".")+ if len(ascii_host) > 253 or any(+ not label+ or len(label) > 63+ or label.startswith("-")+ or label.endswith("-")+ or not all(char.isalnum() or char == "-" for char in label)+ for label in labels+ ):+ raise ValueError("A valid IMAP hostname is required")+ return ascii_host+++def _is_public_address(address: str) -> bool:+ """Return whether an address is globally routable unicast."""+ parsed = ipaddress.ip_address(address)+ if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped is not None:+ parsed = parsed.ipv4_mapped+ # ``is_global`` alone is insufficient: Python classifies multicast ranges+ # as global even though they are not valid remote unicast endpoints.+ return parsed.is_global and not any(+ (+ parsed.is_multicast,+ parsed.is_reserved,+ parsed.is_unspecified,+ parsed.is_loopback,+ parsed.is_link_local,+ parsed.is_private,+ )+ )+++def validate_imap_endpoint(host: str, port: int) -> tuple[str, tuple[str, ...]]:+ """Resolve a public IMAP endpoint and return addresses safe to pin.++ All resolved addresses must be globally routable. Returning the resolved+ addresses lets the socket layer pin one of the checked IPs, closing the+ DNS-rebinding gap between policy validation and connection establishment.+ """+ normalized = _normalize_hostname(host)+ lowered = normalized.lower()+ if (+ lowered in _BLOCKED_HOSTNAMES+ or lowered.endswith(".localhost")+ or lowered.endswith(".local")+ or lowered.endswith(".internal")+ ):+ raise ValueError("Private or local IMAP hosts are not allowed")++ try:+ resolved = socket.getaddrinfo(+ normalized,+ port,+ socket.AF_UNSPEC,+ socket.SOCK_STREAM,+ )+ except socket.gaierror as exc:+ raise ValueError(f"Unable to resolve IMAP host '{normalized}'") from exc++ addresses: list[str] = []+ for _, _, _, _, sockaddr in resolved:+ address = sockaddr[0]+ if not _is_public_address(address):+ raise ValueError("Private or non-public IMAP addresses are not allowed")+ if address not in addresses:+ addresses.append(address)+ if not addresses:+ raise ValueError(f"Unable to resolve IMAP host '{normalized}'")+ return normalized, tuple(addresses)+++def _create_pinned_socket(address: str, port: int, timeout: float | None):+ """Create a socket to an already policy-checked IP literal."""+ if timeout is not None and not timeout:+ raise ValueError("Non-blocking sockets are not supported")+ if timeout is None:+ return socket.create_connection((address, port))+ return socket.create_connection((address, port), timeout)+++class _PinnedIMAP4(imaplib.IMAP4):+ """Plain IMAP transport pinned to a previously validated address."""++ def __init__(+ self,+ host: str,+ port: int,+ pinned_address: str,+ *,+ timeout: float | None = None,+ ) -> None:+ self._pinned_address = pinned_address+ super().__init__(host, port, timeout)++ def _create_socket(self, timeout):+ return _create_pinned_socket(self._pinned_address, self.port, timeout)+++class _PinnedIMAP4SSL(imaplib.IMAP4_SSL):+ """Implicit-TLS IMAP transport with DNS-pinned TCP and hostname TLS."""++ def __init__(+ self,+ host: str,+ port: int,+ pinned_address: str,+ *,+ ssl_context: ssl.SSLContext,+ timeout: float | None = None,+ ) -> None:+ self._pinned_address = pinned_address+ super().__init__(host, port, ssl_context=ssl_context, timeout=timeout)++ def _create_socket(self, timeout):+ sock = _create_pinned_socket(self._pinned_address, self.port, timeout)+ return self.ssl_context.wrap_socket(sock, server_hostname=self.host)+++def resolve_imap_host(email_address: str) -> str:+ """Resolve the IMAP host for an email address from its domain."""+ domain = email_address.rsplit("@", 1)[-1].lower() if "@" in email_address else ""+ if not domain:+ return ""+ return _PROVIDER_HOSTS.get(domain, f"imap.{domain}")++[email protected]("imap")+class IMAPConnector(GmailIMAPConnector):+ """Generic IMAP connector for any email provider."""++ connector_id = "imap"+ display_name = "Email (IMAP)"+ _default_imap_host = ""++ def __init__(+ self,+ email_address: str = "",+ app_password: str = "",+ credentials_path: str = "",+ *,+ imap_host: str = "",+ imap_port: int | None = None,+ imap_security: str = "",+ max_messages: Optional[int] = None,+ ) -> None:+ super().__init__(+ email_address,+ app_password,+ credentials_path or _DEFAULT_CREDENTIALS_PATH,+ imap_host=imap_host,+ imap_port=imap_port,+ imap_security=imap_security,+ max_messages=max_messages,+ )++ def auth_url(self) -> str:+ return ""++ def _host_for_email(self, email_address: str) -> str:+ return resolve_imap_host(email_address)++ def _make_imap_client(self, host: str, port: int, security: str) -> imaplib.IMAP4:+ """Connect only to a DNS-pinned, globally routable endpoint."""+ normalized_host, addresses = validate_imap_endpoint(host, port)+ tls_context = ssl.create_default_context()+ last_error: Exception | None = None+ for address in addresses:+ try:+ if security == _SECURITY_TLS:+ return _PinnedIMAP4SSL(+ normalized_host,+ port,+ address,+ ssl_context=tls_context,+ timeout=_IMAP_TIMEOUT_SECONDS,+ )++ if security != _SECURITY_STARTTLS:+ raise ValueError("IMAP security must be 'tls' or 'starttls'")+ imap = _PinnedIMAP4(+ normalized_host,+ port,+ address,+ timeout=_IMAP_TIMEOUT_SECONDS,+ )+ try:+ status, _ = imap.starttls(ssl_context=tls_context)+ if status != "OK":+ raise imaplib.IMAP4.error("IMAP server rejected STARTTLS")+ return imap+ except Exception:+ self._close_imap(imap)+ raise+ except (imaplib.IMAP4.error, OSError) as exc:+ last_error = exc++ if last_error is not None:+ raise last_error+ raise OSError(f"Unable to connect to IMAP host '{normalized_host}'")++ def sync(+ self,+ *,+ since: Optional[datetime] = None,+ cursor: Optional[str] = None,+ ) -> Iterator[Document]:+ for doc in super().sync(since=since, cursor=cursor):+ doc.source = self.connector_id+ doc.doc_id = doc.doc_id.replace("gmail:", f"{self.connector_id}:", 1)+ # A generic mailbox must not link every result to Gmail.+ doc.url = ""+ yield doc++ def mcp_tools(self) -> list:+ """Do not expose Gmail-branded live tools for non-Gmail mailboxes."""+ return []diff --git a/src/openjarvis/server/connectors_router.py b/src/openjarvis/server/connectors_router.pyindex 2e95ec6d..367865b1 100644--- a/src/openjarvis/server/connectors_router.py+++ b/src/openjarvis/server/connectors_router.py@@ -81,6 +81,9 @@ try:code: Optional[str] = Noneemail: Optional[str] = Nonepassword: Optional[str] = None+ host: Optional[str] = None+ port: Optional[int] = None+ security: Optional[str] = Noneexcept ImportError:ConnectRequest = None # type: ignore[assignment,misc]@@ -409,8 +412,33 @@ def create_connectors_router():directive = _maybe_oauth_client_pair(connector_id, req)if directive is not None:return directive- if req.code:- instance.handle_callback(req.code)+ if (+ req.email+ and req.password+ and hasattr(instance, "connect_with_credentials")+ ):+ from starlette.concurrency import run_in_threadpool++ connection_options: Dict[str, Any] = {}+ # Only the generic IMAP connector accepts a user-selected+ # network endpoint. Its connector implementation applies+ # the server-side public-address policy and DNS pinning.+ if connector_id == "imap":+ connection_options = {+ "imap_host": req.host or "",+ "imap_port": req.port,+ "imap_security": req.security or "",+ }+ await run_in_threadpool(+ instance.connect_with_credentials,+ req.email,+ req.password,+ **connection_options,+ )+ elif req.code:+ from starlette.concurrency import run_in_threadpool++ await run_in_threadpool(instance.handle_callback, req.code)elif req.token:# A credential pasted into the ``token`` field. Connectors# that accept a pre-existing access token expose ``_token``diff --git a/tests/connectors/test_imap.py b/tests/connectors/test_imap.pynew file mode 100644index 00000000..a01cb08b--- /dev/null+++ b/tests/connectors/test_imap.py@@ -0,0 +1,543 @@+"""Tests for the generic IMAP connector and its network boundary."""++from __future__ import annotations++import json+import socket+from imaplib import IMAP4+from pathlib import Path+from unittest.mock import MagicMock, patch++import pytest++from openjarvis.connectors.imap import (+ IMAPConnector,+ _create_pinned_socket,+ resolve_imap_host,+ validate_imap_endpoint,+)+from openjarvis.connectors.oauth import load_tokens+from openjarvis.core.registry import ConnectorRegistry+++def _message(+ *,+ subject: bytes = b"Test Email",+ message_id: bytes = b"<[email protected]>",+) -> bytes:+ return (+ b"From: [email protected]\r\n"+ b"To: [email protected]\r\n"+ b"Subject: " + subject + b"\r\n"+ b"Date: Mon, 01 Jan 2024 00:00:00 +0000\r\n"+ b"Message-ID: " + message_id + b"\r\n\r\n"+ b"Hello from IMAP test"+ )+++def _authenticated_client(uidvalidity: int = 777) -> MagicMock:+ client = MagicMock()+ client.login.return_value = ("OK", [])+ client.select.return_value = ("OK", [])+ client.response.return_value = ("UIDVALIDITY", [str(uidvalidity).encode("ascii")])+ client.logout.return_value = ("OK", [])+ return client+++def _sync_client(raw_email: bytes | None = None) -> MagicMock:+ client = _authenticated_client()+ raw_email = raw_email or _message()++ def uid(command: str, *args):+ if command == "SEARCH":+ return "OK", [b"41"]+ if command == "FETCH":+ assert args == (b"41", "(RFC822)")+ return "OK", [(b"41 (RFC822 {1})", raw_email)]+ raise AssertionError(f"Unexpected UID command: {command}")++ client.uid.side_effect = uid+ return client+++def _connect(connector: IMAPConnector, client: MagicMock | None = None) -> MagicMock:+ client = client or _authenticated_client()+ with patch.object(connector, "_make_imap_client", return_value=client):+ connector.handle_callback("[email protected]:pass")+ return client+++def test_imap_registered() -> None:+ ConnectorRegistry.register_value("imap", IMAPConnector)+ assert ConnectorRegistry.contains("imap")+ cls = ConnectorRegistry.get("imap")+ assert cls.connector_id == "imap"+ assert cls.display_name == "Email (IMAP)"+++def test_resolve_known_provider_host() -> None:+ assert resolve_imap_host("[email protected]") == "imap.gmail.com"+ assert resolve_imap_host("[email protected]") == "outlook.office365.com"+ assert resolve_imap_host("[email protected]") == "imap.fastmail.com"+++def test_resolve_unknown_domain_falls_back() -> None:+ assert resolve_imap_host("[email protected]") == "imap.example.org"+ assert resolve_imap_host("not-an-email") == ""+++def test_callback_validates_before_persisting(tmp_path: Path) -> None:+ creds_path = str(tmp_path / "imap.json")+ conn = IMAPConnector(credentials_path=creds_path)+ client = _authenticated_client()++ with patch.object(conn, "_make_imap_client", return_value=client) as make_client:+ conn.handle_callback("[email protected]:mypassword123")++ make_client.assert_called_once_with("imap.example.org", 993, "tls")+ client.login.assert_called_once_with("[email protected]", "mypassword123")+ client.select.assert_called_once_with("INBOX", readonly=True)+ client.logout.assert_called_once()+ tokens = load_tokens(creds_path)+ assert tokens == {+ "email": "[email protected]",+ "password": "mypassword123",+ "imap_host": "imap.example.org",+ "imap_port": 993,+ "imap_security": "tls",+ "validated": True,+ }+ assert conn.is_connected() is True+++def test_failed_login_is_not_persisted_or_connected(tmp_path: Path) -> None:+ creds_path = str(tmp_path / "imap.json")+ conn = IMAPConnector(credentials_path=creds_path)+ client = _authenticated_client()+ client.login.side_effect = IMAP4.error("bad credentials")++ with (+ patch.object(conn, "_make_imap_client", return_value=client),+ pytest.raises(IMAP4.error, match="bad credentials"),+ ):+ conn.handle_callback("[email protected]:wrong")++ assert load_tokens(creds_path) is None+ assert conn.is_connected() is False+ client.logout.assert_called_once()+++def test_missing_uidvalidity_is_not_persisted_or_connected(tmp_path: Path) -> None:+ creds_path = str(tmp_path / "imap.json")+ conn = IMAPConnector(credentials_path=creds_path)+ client = _authenticated_client()+ client.response.return_value = ("UIDVALIDITY", [None])++ with (+ patch.object(conn, "_make_imap_client", return_value=client),+ pytest.raises(IMAP4.error, match="UIDVALIDITY"),+ ):+ conn.handle_callback("[email protected]:secret")++ assert load_tokens(creds_path) is None+ assert conn.is_connected() is False+ client.logout.assert_called_once()+++def test_unvalidated_legacy_credentials_are_not_connected(tmp_path: Path) -> None:+ creds_path = tmp_path / "imap.json"+ creds_path.write_text(+ json.dumps({"email": "[email protected]", "password": "old"}),+ encoding="utf-8",+ )+ assert IMAPConnector(credentials_path=str(creds_path)).is_connected() is False+++def test_direct_credentials_do_not_inherit_stored_host_or_validation(+ tmp_path: Path,+) -> None:+ creds_path = tmp_path / "imap.json"+ creds_path.write_text(+ json.dumps(+ {+ "email": "[email protected]",+ "password": "old",+ "imap_host": "old.example.org",+ "imap_port": 1143,+ "imap_security": "starttls",+ "validated": True,+ }+ ),+ encoding="utf-8",+ )+ conn = IMAPConnector(+ email_address="[email protected]",+ app_password="new",+ credentials_path=str(creds_path),+ )++ assert conn.is_connected() is False+ assert conn._resolve_connection()[2:5] == ("imap.gmail.com", 993, "tls")+++def test_explicit_starttls_settings_are_validated_and_persisted(+ tmp_path: Path,+) -> None:+ conn = IMAPConnector(credentials_path=str(tmp_path / "imap.json"))+ client = _authenticated_client()++ with patch.object(conn, "_make_imap_client", return_value=client) as make_client:+ conn.connect_with_credentials(+ "[email protected]",+ "secret",+ imap_host="mail.custom.example",+ imap_port=1143,+ imap_security="STARTTLS",+ )++ make_client.assert_called_once_with("mail.custom.example", 1143, "starttls")+ tokens = load_tokens(conn._credentials_path)+ assert tokens is not None+ assert tokens["imap_host"] == "mail.custom.example"+ assert tokens["imap_port"] == 1143+ assert tokens["imap_security"] == "starttls"+++ ("port", "security", "message"),+ [+ (0, "tls", "port"),+ (65536, "tls", "port"),+ (993, "plaintext", "security"),+ ],+)+def test_invalid_transport_settings_fail_before_connecting(+ port: int, security: str, message: str, tmp_path: Path+) -> None:+ conn = IMAPConnector(credentials_path=str(tmp_path / "imap.json"))+ with (+ patch.object(conn, "_make_imap_client") as make_client,+ pytest.raises(ValueError, match=message),+ ):+ conn.connect_with_credentials(+ "[email protected]",+ "secret",+ imap_port=port,+ imap_security=security,+ )+ make_client.assert_not_called()+++ ("host", "address"),+ [+ ("localhost", None),+ ("metadata.google.internal", None),+ ("mail.local", None),+ ("127.0.0.1", "127.0.0.1"),+ ("10.0.0.8", "10.0.0.8"),+ ("169.254.169.254", "169.254.169.254"),+ ("::1", "::1"),+ ("100.64.0.1", "100.64.0.1"),+ ("224.0.0.1", "224.0.0.1"),+ ("239.255.255.250", "239.255.255.250"),+ ("ff02::1", "ff02::1"),+ ("::ffff:224.0.0.1", "::ffff:224.0.0.1"),+ ("0.0.0.0", "0.0.0.0"),+ ("::", "::"),+ ("240.0.0.1", "240.0.0.1"),+ ],+)+def test_endpoint_policy_rejects_local_and_nonpublic_hosts(+ host: str, address: str | None+) -> None:+ if address is None:+ with pytest.raises(ValueError, match="Private|local"):+ validate_imap_endpoint(host, 993)+ return+ family = socket.AF_INET6 if ":" in address else socket.AF_INET+ sockaddr = (address, 993, 0, 0) if family == socket.AF_INET6 else (address, 993)+ with (+ patch(+ "openjarvis.connectors.imap.socket.getaddrinfo",+ return_value=[(family, socket.SOCK_STREAM, 6, "", sockaddr)],+ ),+ pytest.raises(ValueError, match="non-public"),+ ):+ validate_imap_endpoint(host, 993)+++def test_endpoint_policy_rejects_mixed_public_private_dns() -> None:+ answers = [+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 993)),+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 993)),+ ]+ with (+ patch("openjarvis.connectors.imap.socket.getaddrinfo", return_value=answers),+ pytest.raises(ValueError, match="non-public"),+ ):+ validate_imap_endpoint("mail.example.com", 993)+++def test_endpoint_policy_returns_only_checked_public_addresses() -> None:+ answers = [+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 993)),+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 993)),+ (+ socket.AF_INET6,+ socket.SOCK_STREAM,+ 6,+ "",+ ("2606:2800:220:1:248:1893:25c8:1946", 993, 0, 0),+ ),+ ]+ with patch("openjarvis.connectors.imap.socket.getaddrinfo", return_value=answers):+ assert validate_imap_endpoint("MAIL.Example.COM.", 993) == (+ "mail.example.com",+ ("93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"),+ )+++def test_endpoint_policy_fails_closed_on_dns_error() -> None:+ with (+ patch(+ "openjarvis.connectors.imap.socket.getaddrinfo",+ side_effect=socket.gaierror("no result"),+ ),+ pytest.raises(ValueError, match="Unable to resolve"),+ ):+ validate_imap_endpoint("mail.example.com", 993)+++def test_pinned_socket_connects_to_checked_ip_not_hostname() -> None:+ sentinel = object()+ with patch(+ "openjarvis.connectors.imap.socket.create_connection",+ return_value=sentinel,+ ) as create_connection:+ assert _create_pinned_socket("93.184.216.34", 993, 30) is sentinel+ create_connection.assert_called_once_with(("93.184.216.34", 993), 30)+++def test_starttls_is_required_before_login() -> None:+ conn = IMAPConnector()+ client = _authenticated_client()+ client.starttls.return_value = ("OK", [])+ with (+ patch(+ "openjarvis.connectors.imap.validate_imap_endpoint",+ return_value=("mail.example.com", ("93.184.216.34",)),+ ),+ patch("openjarvis.connectors.imap._PinnedIMAP4", return_value=client),+ ):+ assert conn._make_imap_client("mail.example.com", 143, "starttls") is client+ client.starttls.assert_called_once()+++def test_implicit_tls_uses_checked_ip_and_original_hostname() -> None:+ conn = IMAPConnector()+ client = _authenticated_client()+ context = MagicMock()+ with (+ patch(+ "openjarvis.connectors.imap.validate_imap_endpoint",+ return_value=("mail.example.com", ("93.184.216.34",)),+ ),+ patch(+ "openjarvis.connectors.imap.ssl.create_default_context",+ return_value=context,+ ),+ patch(+ "openjarvis.connectors.imap._PinnedIMAP4SSL",+ return_value=client,+ ) as pinned_tls,+ ):+ assert conn._make_imap_client("mail.example.com", 993, "tls") is client++ pinned_tls.assert_called_once_with(+ "mail.example.com",+ 993,+ "93.184.216.34",+ ssl_context=context,+ timeout=30,+ )+++def test_sync_uses_uids_and_generic_document_identity(tmp_path: Path) -> None:+ conn = IMAPConnector(+ email_address="[email protected]",+ app_password="secret",+ credentials_path=str(tmp_path / "imap.json"),+ )+ client = _sync_client()++ with patch.object(conn, "_make_imap_client", return_value=client):+ docs = list(conn.sync())++ assert client.uid.call_args_list[0].args == ("SEARCH", None, "ALL")+ assert client.uid.call_args_list[1].args == ("FETCH", b"41", "(RFC822)")+ client.search.assert_not_called()+ client.fetch.assert_not_called()+ assert len(docs) == 1+ assert docs[0].source == "imap"+ assert docs[0].doc_id == "imap:<[email protected]>"+ assert docs[0].metadata["imap_uid"] == "41"+ assert docs[0].metadata["imap_uidvalidity"] == "777"+ assert docs[0].title == "Test Email"+ assert docs[0].url == ""+ client.logout.assert_called_once()+++def test_missing_message_id_falls_back_to_uid() -> None:+ conn = IMAPConnector(email_address="[email protected]", app_password="secret")+ raw = _message(message_id=b"")+ client = _sync_client(raw)+ with patch.object(conn, "_make_imap_client", return_value=client):+ docs = list(conn.sync())+ assert docs[0].doc_id == "imap:777:41"+++def test_generator_close_logs_out() -> None:+ conn = IMAPConnector(+ email_address="[email protected]",+ app_password="secret",+ max_messages=1,+ )+ client = _sync_client()++ with patch.object(conn, "_make_imap_client", return_value=client):+ generator = conn.sync()+ next(generator)+ generator.close()++ client.logout.assert_called_once()+++def test_sync_failed_login_closes_socket_and_marks_disconnected() -> None:+ conn = IMAPConnector(email_address="[email protected]", app_password="bad")+ client = _authenticated_client()+ client.login.side_effect = IMAP4.error("bad credentials")++ with patch.object(conn, "_make_imap_client", return_value=client):+ assert list(conn.sync()) == []++ assert conn.is_connected() is False+ client.logout.assert_called_once()+++def test_generic_imap_does_not_advertise_gmail_tools() -> None:+ assert IMAPConnector().mcp_tools() == []+++def test_sync_handles_raw_8bit_headers() -> None:+ conn = IMAPConnector(email_address="[email protected]", app_password="pass")+ raw_email = _message(subject=b"caf\xe9 meeting")+ client = _sync_client(raw_email)++ with patch.object(conn, "_make_imap_client", return_value=client):+ docs = list(conn.sync())++ assert len(docs) == 1+ document = docs[0]+ assert isinstance(document.author, str)+ assert isinstance(document.title, str)+ assert all(isinstance(participant, str) for participant in document.participants)+ json.dumps(+ {+ "author": document.author,+ "title": document.title,+ "participants": document.participants,+ **document.metadata,+ }+ )+++def test_sync_reconnects_by_uid_on_dropped_connection() -> None:+ conn = IMAPConnector(email_address="[email protected]", app_password="pass")+ first = _authenticated_client()++ def first_uid(command: str, *args):+ if command == "SEARCH":+ return "OK", [b"41"]+ raise OSError("[SSL: BAD_LENGTH] bad length")++ first.uid.side_effect = first_uid+ second = _authenticated_client()++ def second_uid(command: str, *args):+ assert command == "FETCH"+ assert args == (b"41", "(RFC822)")+ return "OK", [(b"41 (RFC822 {1})", _message())]++ second.uid.side_effect = second_uid++ with patch.object(conn, "_make_imap_client", side_effect=[first, second]):+ docs = list(conn.sync())++ assert len(docs) == 1+ assert docs[0].metadata["imap_uid"] == "41"+ assert docs[0].metadata["imap_uidvalidity"] == "777"+ first.logout.assert_called_once()+ second.logout.assert_called_once()+++def test_sync_stops_if_uidvalidity_changes_during_reconnect() -> None:+ conn = IMAPConnector(email_address="[email protected]", app_password="pass")+ first = _authenticated_client(uidvalidity=777)++ def first_uid(command: str, *args):+ if command == "SEARCH":+ return "OK", [b"41"]+ raise OSError("connection dropped")++ first.uid.side_effect = first_uid+ second = _authenticated_client(uidvalidity=888)++ with patch.object(conn, "_make_imap_client", side_effect=[first, second]):+ assert list(conn.sync()) == []++ second.uid.assert_not_called()+ first.logout.assert_called_once()+ second.logout.assert_called_once()+++def test_server_rejects_private_custom_host_without_saving_credentials(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ fastapi = pytest.importorskip("fastapi")+ testclient = pytest.importorskip("fastapi.testclient")+ from openjarvis.server import connectors_router++ class HermeticIMAPConnector(IMAPConnector):+ def __init__(self) -> None:+ super().__init__(credentials_path=str(tmp_path / "imap.json"))++ ConnectorRegistry.register_value("imap", HermeticIMAPConnector)+ connectors_router._instances.clear()+ monkeypatch.setattr(+ "openjarvis.connectors.imap.socket.getaddrinfo",+ lambda *args, **kwargs: [+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 1993))+ ],+ )+ app = fastapi.FastAPI()+ app.include_router(connectors_router.create_connectors_router())++ try:+ response = testclient.TestClient(app).post(+ "/v1/connectors/imap/connect",+ json={+ "email": "[email protected]",+ "password": "secret",+ "host": "mail.attacker.example",+ "port": 1993,+ "security": "tls",+ },+ )+ finally:+ ConnectorRegistry.clear()+ connectors_router._instances.clear()++ assert response.status_code == 400+ assert "non-public" in response.json()["detail"]+ assert not (tmp_path / "imap.json").exists()diff --git a/tests/connectors/test_outlook.py b/tests/connectors/test_outlook.pyindex c6799c97..fa9a4a86 100644--- a/tests/connectors/test_outlook.py+++ b/tests/connectors/test_outlook.py@@ -31,30 +31,40 @@ def test_outlook_auth_url() -> None:def test_outlook_handle_callback(tmp_path: Path) -> None:creds_path = str(tmp_path / "outlook.json")conn = OutlookConnector(credentials_path=creds_path)- conn.handle_callback("[email protected]:mypassword123")+ mock_imap = MagicMock()+ mock_imap.login.return_value = ("OK", [])+ mock_imap.select.return_value = ("OK", [])+ mock_imap.response.return_value = ("UIDVALIDITY", [b"777"])+ with patch.object(conn, "_make_imap_client", return_value=mock_imap):+ conn.handle_callback("[email protected]:mypassword123")tokens = load_tokens(creds_path)assert tokens is not Noneassert tokens["email"] == "[email protected]"assert tokens["password"] == "mypassword123"+ assert tokens["validated"] is Truedef test_outlook_is_connected(tmp_path: Path) -> None:creds_path = str(tmp_path / "outlook.json")conn = OutlookConnector(credentials_path=creds_path)assert conn.is_connected() is False- conn.handle_callback("[email protected]:pass")+ mock_imap = MagicMock()+ mock_imap.login.return_value = ("OK", [])+ mock_imap.select.return_value = ("OK", [])+ mock_imap.response.return_value = ("UIDVALIDITY", [b"777"])+ with patch.object(conn, "_make_imap_client", return_value=mock_imap):+ conn.handle_callback("[email protected]:pass")assert conn.is_connected() is Truedef test_outlook_sync_source_is_outlook(tmp_path: Path) -> None:creds_path = str(tmp_path / "outlook.json")conn = OutlookConnector(credentials_path=creds_path)- conn.handle_callback("[email protected]:pass")mock_imap = MagicMock()mock_imap.login.return_value = ("OK", [])mock_imap.select.return_value = ("OK", [])- mock_imap.search.return_value = ("OK", [b"1"])+ mock_imap.response.return_value = ("UIDVALIDITY", [b"777"])raw_email = (b"From: [email protected]\r\n"@@ -65,15 +75,19 @@ def test_outlook_sync_source_is_outlook(tmp_path: Path) -> None:b"\r\n"b"Hello from Outlook test")- mock_imap.fetch.return_value = ("OK", [(b"1", raw_email)])+ mock_imap.uid.side_effect = [+ ("OK", [b"1"]),+ ("OK", [(b"1", raw_email)]),+ ]mock_imap.logout.return_value = ("OK", [])- with patch("openjarvis.connectors.gmail_imap.imaplib") as mock_imaplib:- mock_imaplib.IMAP4_SSL.return_value = mock_imap- mock_imaplib.IMAP4 = type(mock_imap)+ with patch.object(conn, "_make_imap_client", return_value=mock_imap):+ conn.handle_callback("[email protected]:pass")docs = list(conn.sync())assert len(docs) == 1assert docs[0].source == "outlook"assert docs[0].doc_id.startswith("outlook:")assert docs[0].title == "Test Email"+ assert mock_imap.uid.call_args_list[0].args == ("SEARCH", None, "ALL")+ assert mock_imap.uid.call_args_list[1].args == ("FETCH", b"1", "(RFC822)")