Skip to content

Personal AI, On Personal Devices

Python

feat(connectors): generic IMAP email connector for any provider (#518) (#586)

Aditya Malikb266f16950d3
diff --git a/frontend/src/components/setup/SourceConnectFlow.tsx b/frontend/src/components/setup/SourceConnectFlow.tsx
index d33e9f6e..cc55fce5 100644
--- a/frontend/src/components/setup/SourceConnectFlow.tsx
+++ b/frontend/src/components/setup/SourceConnectFlow.tsx
@@ -283,9 +283,11 @@ function StepByStepPanel({
onSkip: () => void;
isConnecting: boolean;
}) {
- const [inputs, setInputs] = useState<Record<string, string>>({});
const steps = connector.steps || [];
const fields = connector.inputFields || [];
+ const [inputs, setInputs] = useState<Record<string, string>>(() =>
+ Object.fromEntries(fields.map((field) => [field.name, field.defaultValue || ''])),
+ );
const updateInput = (name: string, value: string) => {
setInputs((prev) => ({ ...prev, [name]: value }));
@@ -298,6 +300,11 @@ function StepByStepPanel({
else if (field.name === 'password') req.password = inputs.password;
else if (field.name === 'token') req.token = inputs.token;
else if (field.name === 'path') req.path = inputs.path;
+ else if (field.name === 'host' && inputs.host?.trim()) req.host = inputs.host.trim();
+ else if (field.name === 'port' && inputs.port?.trim()) req.port = Number(inputs.port);
+ else if (field.name === 'security' && (inputs.security === 'tls' || inputs.security === 'starttls')) {
+ req.security = inputs.security;
+ }
}
// For email+password connectors, also set token as email:password
if (req.email && req.password) {
@@ -310,7 +317,9 @@ function StepByStepPanel({
onConnect(req);
};
- const allFilled = fields.every((f) => inputs[f.name]?.trim());
+ const allFilled = fields
+ .filter((field) => field.required !== false)
+ .every((field) => inputs[field.name]?.trim());
return (
<div style={{ padding: '0 4px' }}>
@@ -378,24 +387,49 @@ function StepByStepPanel({
marginBottom: 10,
}}>
{fields.map((field) => (
- <input
- key={field.name}
- value={inputs[field.name] || ''}
- onChange={(e) => updateInput(field.name, e.target.value)}
- placeholder={field.placeholder}
- type={field.type || 'text'}
- style={{
- width: '100%',
- padding: '8px 10px',
- background: 'var(--color-bg-secondary)',
- border: '1px solid var(--color-border)',
- borderRadius: 4,
- color: 'var(--color-text)',
- fontSize: 13,
- marginBottom: 8,
- boxSizing: 'border-box',
- }}
- />
+ field.type === 'select' ? (
+ <select
+ key={field.name}
+ value={inputs[field.name] || field.defaultValue || ''}
+ onChange={(e) => updateInput(field.name, e.target.value)}
+ aria-label={field.placeholder}
+ style={{
+ width: '100%',
+ padding: '8px 10px',
+ background: 'var(--color-bg-secondary)',
+ border: '1px solid var(--color-border)',
+ borderRadius: 4,
+ color: 'var(--color-text)',
+ fontSize: 13,
+ marginBottom: 8,
+ boxSizing: 'border-box',
+ }}
+ >
+ {(field.options || []).map((option) => (
+ <option key={option.value} value={option.value}>{option.label}</option>
+ ))}
+ </select>
+ ) : (
+ <input
+ key={field.name}
+ value={inputs[field.name] || ''}
+ onChange={(e) => updateInput(field.name, e.target.value)}
+ placeholder={field.placeholder}
+ type={field.type || 'text'}
+ required={field.required !== false}
+ style={{
+ width: '100%',
+ padding: '8px 10px',
+ background: 'var(--color-bg-secondary)',
+ border: '1px solid var(--color-border)',
+ borderRadius: 4,
+ color: 'var(--color-text)',
+ fontSize: 13,
+ marginBottom: 8,
+ boxSizing: 'border-box',
+ }}
+ />
+ )
))}
</div>
)}
@@ -549,6 +583,7 @@ export function SourceConnectFlow({
</div>
) : activeCard.steps ? (
<StepByStepPanel
+ key={activeCard.connector_id}
connector={activeCard}
onConnect={(req) => handleConnect(activeEntry.id, req)}
onSkip={() => handleSkip(activeEntry.id)}
diff --git a/frontend/src/pages/DataSourcesPage.tsx b/frontend/src/pages/DataSourcesPage.tsx
index 795acf02..0ed27349 100644
--- a/frontend/src/pages/DataSourcesPage.tsx
+++ b/frontend/src/pages/DataSourcesPage.tsx
@@ -23,7 +23,7 @@ import {
} from 'lucide-react';
import type { LucideIcon } from 'lucide-react';
import { SOURCE_CATALOG } from '../types/connectors';
-import type { ConnectRequest } from '../types/connectors';
+import type { ConnectRequest, ConnectorMeta } from '../types/connectors';
import { listConnectors, connectSource, disconnectSource, getSyncStatus, triggerSync, startServerOAuth } from '../lib/connectors-api';
import type { SyncStatus } from '../types/connectors';
@@ -36,16 +36,20 @@ function InlineConnectForm({
loading,
onSubmit,
}: {
- fields: Array<{ name: string; placeholder: string; type?: string }>;
+ fields: NonNullable<ConnectorMeta['inputFields']>;
loading: boolean;
onSubmit: (req: ConnectRequest) => void;
}) {
- const [inputs, setInputs] = useState<Record<string, string>>({});
+ const [inputs, setInputs] = useState<Record<string, string>>(() =>
+ Object.fromEntries(fields.map((field) => [field.name, field.defaultValue || ''])),
+ );
const update = (name: string, value: string) =>
setInputs((p) => ({ ...p, [name]: value }));
- const allFilled = fields.every((f) => inputs[f.name]?.trim());
+ const allFilled = fields
+ .filter((field) => field.required !== false)
+ .every((field) => inputs[field.name]?.trim());
const submit = () => {
const req: ConnectRequest = {};
@@ -54,6 +58,11 @@ function InlineConnectForm({
else if (f.name === 'password') req.password = inputs.password;
else if (f.name === 'token') req.token = inputs.token;
else if (f.name === 'path') req.path = inputs.path;
+ else if (f.name === 'host' && inputs.host?.trim()) req.host = inputs.host.trim();
+ else if (f.name === 'port' && inputs.port?.trim()) req.port = Number(inputs.port);
+ else if (f.name === 'security' && (inputs.security === 'tls' || inputs.security === 'starttls')) {
+ req.security = inputs.security;
+ }
}
if (req.email && req.password) {
req.token = `${req.email}:${req.password}`;
@@ -66,21 +75,43 @@ function InlineConnectForm({
return (
<div>
{fields.map((f) => (
- <input
- key={f.name}
- value={inputs[f.name] || ''}
- onChange={(e) => update(f.name, e.target.value)}
- placeholder={f.placeholder}
- type={f.type || 'text'}
- style={{
- width: '100%', padding: '7px 10px',
- background: 'var(--color-bg)',
- border: '1px solid var(--color-border)',
- borderRadius: 4, color: 'var(--color-text)',
- fontSize: 12, marginBottom: 6,
- boxSizing: 'border-box',
- }}
- />
+ f.type === 'select' ? (
+ <select
+ key={f.name}
+ value={inputs[f.name] || f.defaultValue || ''}
+ onChange={(e) => update(f.name, e.target.value)}
+ aria-label={f.placeholder}
+ style={{
+ width: '100%', padding: '7px 10px',
+ background: 'var(--color-bg)',
+ border: '1px solid var(--color-border)',
+ borderRadius: 4, color: 'var(--color-text)',
+ fontSize: 12, marginBottom: 6,
+ boxSizing: 'border-box',
+ }}
+ >
+ {(f.options || []).map((option) => (
+ <option key={option.value} value={option.value}>{option.label}</option>
+ ))}
+ </select>
+ ) : (
+ <input
+ key={f.name}
+ value={inputs[f.name] || ''}
+ onChange={(e) => update(f.name, e.target.value)}
+ placeholder={f.placeholder}
+ type={f.type || 'text'}
+ required={f.required !== false}
+ style={{
+ width: '100%', padding: '7px 10px',
+ background: 'var(--color-bg)',
+ border: '1px solid var(--color-border)',
+ borderRadius: 4, color: 'var(--color-text)',
+ fontSize: 12, marginBottom: 6,
+ boxSizing: 'border-box',
+ }}
+ />
+ )
))}
<button
onClick={submit}
diff --git a/frontend/src/types/connectors.ts b/frontend/src/types/connectors.ts
index 76091396..146f173f 100644
--- a/frontend/src/types/connectors.ts
+++ b/frontend/src/types/connectors.ts
@@ -18,7 +18,10 @@ export interface ConnectorMeta {
inputFields?: Array<{
name: string;
placeholder: string;
- type?: 'text' | 'password';
+ type?: 'text' | 'password' | 'number' | 'select';
+ required?: boolean;
+ defaultValue?: string;
+ options?: Array<{ value: string; label: string }>;
}>;
}
@@ -53,6 +56,9 @@ export interface ConnectRequest {
code?: string;
email?: string;
password?: string;
+ host?: string;
+ port?: number;
+ security?: 'tls' | 'starttls';
}
/** Response from POST /v1/connectors/{id}/connect.
@@ -120,6 +126,52 @@ export const SOURCE_CATALOG: ConnectorMeta[] = [
{ name: 'password', placeholder: 'App password (xxxx xxxx xxxx xxxx)', type: 'password' },
],
},
+ {
+ connector_id: 'imap',
+ display_name: 'Email (IMAP)',
+ auth_type: 'oauth',
+ category: 'communication',
+ icon: 'Mail',
+ color: 'text-amber-400',
+ description: 'Any IMAP mailbox',
+ unitLabel: 'emails',
+ steps: [
+ {
+ label: 'Generate an app password with your email provider (IMAP must be enabled), then enter your email address and app password. Known providers are detected automatically; use the optional server settings for other providers.',
+ },
+ ],
+ troubleshooting: [
+ 'Most providers require an app password rather than your normal password.',
+ 'Use TLS for implicit TLS (usually port 993), or STARTTLS (usually port 143).',
+ 'For security, private, loopback, link-local, and otherwise non-public server addresses are rejected.',
+ ],
+ inputFields: [
+ { name: 'email', placeholder: '[email protected]', type: 'text' },
+ { name: 'password', placeholder: 'App password', type: 'password' },
+ {
+ name: 'host',
+ placeholder: 'IMAP host (optional; auto-detected)',
+ type: 'text',
+ required: false,
+ },
+ {
+ name: 'port',
+ placeholder: 'Port (optional; 993 or 143)',
+ type: 'number',
+ required: false,
+ },
+ {
+ name: 'security',
+ placeholder: 'Connection security',
+ type: 'select',
+ defaultValue: 'tls',
+ options: [
+ { value: 'tls', label: 'TLS (usually port 993)' },
+ { value: 'starttls', label: 'STARTTLS (usually port 143)' },
+ ],
+ },
+ ],
+ },
{
connector_id: 'slack',
display_name: 'Slack',
diff --git a/src/openjarvis/connectors/__init__.py b/src/openjarvis/connectors/__init__.py
index 74497b3e..3ff9d83a 100644
--- a/src/openjarvis/connectors/__init__.py
+++ b/src/openjarvis/connectors/__init__.py
@@ -78,6 +78,11 @@ try:
except ImportError:
pass
+try:
+ import openjarvis.connectors.imap # noqa: F401
+except ImportError:
+ pass
+
try:
import openjarvis.connectors.gcalendar # noqa: F401
except ImportError:
diff --git a/src/openjarvis/connectors/gmail_imap.py b/src/openjarvis/connectors/gmail_imap.py
index 61836bae..1d00d2a2 100644
--- a/src/openjarvis/connectors/gmail_imap.py
+++ b/src/openjarvis/connectors/gmail_imap.py
@@ -11,9 +11,11 @@ from __future__ import annotations
import email as email_lib
import imaplib
import logging
+import ssl
from datetime import datetime
from email.header import decode_header
from email.utils import parsedate_to_datetime
+from imaplib import IMAP4
from typing import Iterator, List, Optional
from openjarvis.connectors._stubs import BaseConnector, Document, SyncStatus
@@ -25,19 +27,57 @@ from openjarvis.tools._stubs import ToolSpec
logger = logging.getLogger(__name__)
_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "gmail_imap.json")
+_IMAP_TIMEOUT_SECONDS = 30
+_SECURITY_TLS = "tls"
+_SECURITY_STARTTLS = "starttls"
+_UIDVALIDITY_ATTRIBUTE = "_openjarvis_uidvalidity"
+
+
+def _normalize_imap_security(value: str) -> str:
+ """Normalize an explicit encrypted IMAP transport mode."""
+ normalized = (value or _SECURITY_TLS).strip().lower().replace("-", "")
+ if normalized in {"tls", "ssl", "implicit", "implicittls"}:
+ return _SECURITY_TLS
+ if normalized == "starttls":
+ return _SECURITY_STARTTLS
+ raise ValueError("IMAP security must be 'tls' or 'starttls'")
+
+
+def _normalize_imap_port(value: object, security: str) -> int:
+ """Return a valid port, defaulting from the selected transport mode."""
+ if value is None or value == "":
+ return 993 if security == _SECURITY_TLS else 143
+ if isinstance(value, bool):
+ raise ValueError("IMAP port must be an integer between 1 and 65535")
+ try:
+ port = int(value)
+ except (TypeError, ValueError) as exc:
+ raise ValueError("IMAP port must be an integer between 1 and 65535") from exc
+ if not 1 <= port <= 65535:
+ raise ValueError("IMAP port must be an integer between 1 and 65535")
+ return port
+
+def _decode_header(raw: object) -> str:
+ """Decode an email header (str or Header) to a safe plain string.
-def _decode_subject(raw: str) -> str:
- """Decode a possibly-encoded email subject header."""
+ Real mail can carry raw 8-bit bytes in headers, which makes ``msg.get()``
+ return an ``email.header.Header`` (not JSON serializable) and yields bogus
+ charsets like ``unknown-8bit``. Coerce to a string, replacing what can't be
+ decoded.
+ """
if not raw:
return ""
- decoded_parts = decode_header(raw)
- return "".join(
- part.decode(enc or "utf-8", errors="replace")
- if isinstance(part, bytes)
- else part
- for part, enc in decoded_parts
- )
+ out = []
+ for part, enc in decode_header(raw if isinstance(raw, str) else str(raw)):
+ if isinstance(part, bytes):
+ try:
+ out.append(part.decode(enc or "utf-8", errors="replace"))
+ except LookupError:
+ out.append(part.decode("utf-8", errors="replace"))
+ else:
+ out.append(part)
+ return "".join(out)
def _extract_text_body(msg: email_lib.message.Message) -> str:
@@ -73,6 +113,28 @@ def _parse_date(msg: email_lib.message.Message) -> datetime:
return datetime.now()
+def _read_uidvalidity(imap: imaplib.IMAP4) -> str:
+ """Read the UID generation for the currently selected mailbox."""
+ response_type, data = imap.response("UIDVALIDITY")
+ if isinstance(response_type, bytes):
+ response_type = response_type.decode("ascii", errors="replace")
+ if str(response_type).upper() != "UIDVALIDITY" or not data:
+ raise IMAP4.error("IMAP server did not provide UIDVALIDITY")
+
+ raw_value = data[-1]
+ if isinstance(raw_value, bytes):
+ text = raw_value.decode("ascii", errors="strict").strip()
+ else:
+ text = str(raw_value).strip()
+ try:
+ value = int(text)
+ except (TypeError, ValueError) as exc:
+ raise IMAP4.error("IMAP server provided invalid UIDVALIDITY") from exc
+ if value <= 0:
+ raise IMAP4.error("IMAP server provided invalid UIDVALIDITY")
+ return str(value)
+
+
@ConnectorRegistry.register("gmail_imap")
class GmailIMAPConnector(BaseConnector):
"""Gmail connector using IMAP + app password.
@@ -92,12 +154,19 @@ class GmailIMAPConnector(BaseConnector):
credentials_path: str = "",
*,
imap_host: str = "",
+ imap_port: int | None = None,
+ imap_security: str = "",
max_messages: Optional[int] = None,
) -> None:
self._email = email_address
self._password = app_password
self._credentials_path = credentials_path or _DEFAULT_CREDENTIALS_PATH
self._imap_host = imap_host or self._default_imap_host
+ self._imap_port = imap_port
+ self._imap_security = imap_security
+ # None means "use the persisted validation marker". Direct credentials
+ # are not considered connected until a real LOGIN + SELECT succeeds.
+ self._credentials_validated: bool | None = None
# ``None`` means "no cap" — the full inbox is indexed. A positive
# value is still honored for tests that want a bounded scan.
self._max_messages = max_messages
@@ -106,20 +175,53 @@ class GmailIMAPConnector(BaseConnector):
def _resolve_credentials(self) -> tuple[str, str]:
"""Return (email, password) — direct args take priority."""
- if self._email and self._password:
- return self._email, self._password
- tokens = load_tokens(self._credentials_path)
- if tokens:
- return tokens.get("email", ""), tokens.get("password", "")
- return "", ""
+ email_address, password, _, _, _, _ = self._resolve_connection()
+ return email_address, password
+
+ def _host_for_email(self, email_address: str) -> str:
+ """Return the connector's provider host for *email_address*."""
+ return self._default_imap_host
+
+ def _resolve_connection(self) -> tuple[str, str, str, int, str, bool]:
+ """Resolve credentials and transport settings from args or storage."""
+ tokens = load_tokens(self._credentials_path) or {}
+ using_direct_credentials = bool(self._email and self._password)
+ if using_direct_credentials:
+ email_address, password = self._email, self._password
+ stored_host = ""
+ stored_security = ""
+ stored_port = None
+ else:
+ email_address = str(tokens.get("email", ""))
+ password = str(tokens.get("password", ""))
+ stored_host = str(tokens.get("imap_host", ""))
+ stored_security = str(tokens.get("imap_security", ""))
+ stored_port = tokens.get("imap_port")
+
+ host = (
+ self._imap_host or stored_host or self._host_for_email(email_address)
+ ).strip()
+ security = _normalize_imap_security(self._imap_security or stored_security)
+ port = _normalize_imap_port(
+ self._imap_port if self._imap_port is not None else stored_port,
+ security,
+ )
+ if self._credentials_validated is not None:
+ validated = self._credentials_validated
+ elif using_direct_credentials:
+ validated = False
+ else:
+ validated = tokens.get("validated") is True
+ return email_address, password, host, port, security, validated
def is_connected(self) -> bool:
- em, pw = self._resolve_credentials()
- return bool(em and pw)
+ em, pw, host, _, _, validated = self._resolve_connection()
+ return bool(em and pw and host and validated)
def disconnect(self) -> None:
self._email = ""
self._password = ""
+ self._credentials_validated = False
delete_tokens(self._credentials_path)
def auth_url(self) -> str:
@@ -127,18 +229,128 @@ class GmailIMAPConnector(BaseConnector):
def handle_callback(self, code: str) -> None:
# code format: "email:password"
- if ":" in code:
- em, pw = code.split(":", 1)
- save_tokens(
- self._credentials_path,
- {"email": em.strip(), "password": pw.strip()},
- )
- else:
- save_tokens(
- self._credentials_path,
- {"email": "", "password": code.strip()},
+ if ":" not in code:
+ raise ValueError("Expected credentials in 'email:password' format")
+ em, pw = code.split(":", 1)
+ self.connect_with_credentials(em, pw)
+
+ def connect_with_credentials(
+ self,
+ email_address: str,
+ app_password: str,
+ *,
+ imap_host: str = "",
+ imap_port: int | None = None,
+ imap_security: str = "",
+ ) -> None:
+ """Validate LOGIN + INBOX access, then persist the connection."""
+ email_address = email_address.strip()
+ app_password = app_password.strip()
+ if not email_address or not app_password:
+ raise ValueError("Email address and app password are required")
+
+ host = (imap_host or self._host_for_email(email_address)).strip()
+ if not host:
+ raise ValueError("An IMAP host is required")
+ security = _normalize_imap_security(imap_security)
+ port = _normalize_imap_port(imap_port, security)
+
+ imap = self._open_authenticated_imap(
+ email_address,
+ app_password,
+ host,
+ port,
+ security,
+ )
+ self._close_imap(imap)
+
+ save_tokens(
+ self._credentials_path,
+ {
+ "email": email_address,
+ "password": app_password,
+ "imap_host": host,
+ "imap_port": port,
+ "imap_security": security,
+ "validated": True,
+ },
+ )
+ self._email = email_address
+ self._password = app_password
+ self._imap_host = host
+ self._imap_port = port
+ self._imap_security = security
+ self._credentials_validated = True
+
+ def _make_imap_client(self, host: str, port: int, security: str) -> imaplib.IMAP4:
+ """Create a TLS or STARTTLS IMAP client."""
+ tls_context = ssl.create_default_context()
+ if security == _SECURITY_TLS:
+ return imaplib.IMAP4_SSL(
+ host,
+ port,
+ ssl_context=tls_context,
+ timeout=_IMAP_TIMEOUT_SECONDS,
)
+ imap = imaplib.IMAP4(host, port, timeout=_IMAP_TIMEOUT_SECONDS)
+ try:
+ status, _ = imap.starttls(ssl_context=tls_context)
+ if status != "OK":
+ raise IMAP4.error("IMAP server rejected STARTTLS")
+ return imap
+ except Exception:
+ self._close_imap(imap)
+ raise
+
+ def _open_authenticated_imap(
+ self,
+ email_address: str,
+ password: str,
+ host: str,
+ port: int,
+ security: str,
+ ) -> imaplib.IMAP4:
+ """Open, authenticate, and select INBOX on one IMAP connection."""
+ imap = self._make_imap_client(host, port, security)
+ try:
+ status, _ = imap.login(email_address, password)
+ if status != "OK":
+ raise IMAP4.error("IMAP login rejected")
+ status, _ = imap.select("INBOX", readonly=True)
+ if status != "OK":
+ raise IMAP4.error("Unable to select INBOX read-only")
+ # A UID is meaningful only together with this mailbox generation.
+ # Store it on the selected connection so reconnects cannot reuse an
+ # old UID against a reset or replaced mailbox.
+ setattr(imap, _UIDVALIDITY_ATTRIBUTE, _read_uidvalidity(imap))
+ return imap
+ except Exception:
+ self._close_imap(imap)
+ raise
+
+ def _open_imap(self) -> imaplib.IMAP4:
+ """Open an authenticated, INBOX-selected IMAP connection."""
+ em, pw, host, port, security, _ = self._resolve_connection()
+ if not em or not pw or not host:
+ raise ValueError("Complete IMAP credentials are required")
+ imap = self._open_authenticated_imap(em, pw, host, port, security)
+ self._credentials_validated = True
+ return imap
+
+ @staticmethod
+ def _close_imap(imap: imaplib.IMAP4 | None) -> None:
+ """Best-effort logout for normal, failed, and cancelled syncs."""
+ if imap is None:
+ return
+ try:
+ imap.logout()
+ except Exception:
+ try:
+ imap.shutdown()
+ except Exception:
+ pass
+
def sync(
self,
*,
@@ -149,69 +361,124 @@ class GmailIMAPConnector(BaseConnector):
if not em or not pw:
return
- imap = imaplib.IMAP4_SSL(self._imap_host)
try:
- imap.login(em, pw)
- except imaplib.IMAP4.error as exc:
+ imap = self._open_imap()
+ except (IMAP4.error, OSError, ValueError) as exc:
+ if isinstance(exc, (IMAP4.error, ValueError)):
+ self._credentials_validated = False
logger.error("IMAP login failed: %s", exc)
return
- imap.select("INBOX", readonly=True)
-
- # Always SEARCH ALL. IMAP has no native cursor that survives a
- # server restart, so applying the SyncEngine's ``since`` filter
- # during a partial backfill would silently skip the older
- # unprocessed messages. The pipeline-level dedup (_seen_doc_ids
- # set + INSERT OR IGNORE in KnowledgeStore) makes re-scanning
- # already-indexed messages cheap, so resume is correct as long
- # as we keep enumerating the full inbox.
- _, data = imap.search(None, "ALL")
- msg_ids = data[0].split()
- self._items_total = len(msg_ids)
-
- # Newest-first iteration so Deep Research becomes useful while
- # the long tail of older mail finishes indexing in the
- # background. IMAP returns sequence numbers in arrival order
- # (oldest -> newest), so reversing puts the most recent first.
- ordered = list(reversed(msg_ids))
- if self._max_messages is not None and self._max_messages > 0:
- ordered = ordered[: self._max_messages]
synced = 0
-
- for mid in ordered:
- try:
- _, msg_data = imap.fetch(mid, "(RFC822)")
- raw = msg_data[0][1]
- msg = email_lib.message_from_bytes(raw)
- except Exception:
- continue
-
- subject = _decode_subject(msg.get("Subject", ""))
- sender = msg.get("From", "")
- to = msg.get("To", "")
- body = _extract_text_body(msg)
- timestamp = _parse_date(msg)
- message_id = msg.get("Message-ID", mid.decode())
-
- synced += 1
- yield Document(
- doc_id=f"gmail:{message_id}",
- source="gmail",
- doc_type="email",
- content=body,
- title=subject,
- author=sender,
- participants=[a.strip() for a in (to or "").split(",") if a.strip()],
- timestamp=timestamp,
- thread_id=msg.get("In-Reply-To", ""),
- url="https://mail.google.com/mail/u/0/#inbox",
- metadata={
- "message_id": message_id,
- },
- )
-
- imap.logout()
- self._items_synced = synced
+ try:
+ uidvalidity = str(getattr(imap, _UIDVALIDITY_ATTRIBUTE))
+ # SEARCH ALL is deliberate: pipeline dedup makes a complete scan
+ # safe while avoiding gaps after an interrupted backfill.
+ status, data = imap.uid("SEARCH", None, "ALL")
+ if status != "OK" or not data or data[0] is None:
+ logger.error("IMAP UID SEARCH ALL failed: %s", status)
+ return
+ message_uids = data[0].split() if data[0] else []
+ self._items_total = len(message_uids)
+ ordered = list(reversed(message_uids))
+ if self._max_messages is not None and self._max_messages > 0:
+ ordered = ordered[: self._max_messages]
+
+ max_reconnects = 5
+ consecutive_reconnects = 0
+ index = 0
+ while index < len(ordered):
+ message_uid = ordered[index]
+ try:
+ status, msg_data = imap.uid("FETCH", message_uid, "(RFC822)")
+ message_record = next(
+ (
+ item
+ for item in msg_data or []
+ if isinstance(item, tuple)
+ and len(item) > 1
+ and isinstance(item[1], bytes)
+ ),
+ None,
+ )
+ if status != "OK" or message_record is None:
+ index += 1
+ continue
+ raw = message_record[1]
+ msg = email_lib.message_from_bytes(raw)
+ except (IMAP4.abort, OSError) as exc:
+ if consecutive_reconnects >= max_reconnects:
+ logger.error(
+ "IMAP sync stopping after %d reconnect attempts: %s",
+ consecutive_reconnects,
+ exc,
+ )
+ break
+ consecutive_reconnects += 1
+ logger.warning(
+ "IMAP connection dropped (%s); reconnecting %d/%d",
+ exc,
+ consecutive_reconnects,
+ max_reconnects,
+ )
+ self._close_imap(imap)
+ imap = None
+ try:
+ imap = self._open_imap()
+ except (IMAP4.error, OSError, ValueError) as reconnect_exc:
+ logger.error("IMAP reconnect failed: %s", reconnect_exc)
+ break
+ reconnected_uidvalidity = str(getattr(imap, _UIDVALIDITY_ATTRIBUTE))
+ if reconnected_uidvalidity != uidvalidity:
+ logger.error(
+ "IMAP UIDVALIDITY changed from %s to %s during sync; "
+ "stopping before reusing stale UIDs",
+ uidvalidity,
+ reconnected_uidvalidity,
+ )
+ break
+ continue
+ except Exception:
+ index += 1
+ continue
+
+ subject = _decode_header(msg.get("Subject", ""))
+ sender = _decode_header(msg.get("From", ""))
+ to = _decode_header(msg.get("To", ""))
+ body = _extract_text_body(msg)
+ timestamp = _parse_date(msg)
+ uid_text = message_uid.decode("ascii", errors="replace")
+ message_id = _decode_header(msg.get("Message-ID", "")) or (
+ f"{uidvalidity}:{uid_text}"
+ )
+
+ synced += 1
+ index += 1
+ consecutive_reconnects = 0
+ yield Document(
+ doc_id=f"gmail:{message_id}",
+ source="gmail",
+ doc_type="email",
+ content=body,
+ title=subject,
+ author=sender,
+ participants=[
+ address.strip()
+ for address in (to or "").split(",")
+ if address.strip()
+ ],
+ timestamp=timestamp,
+ thread_id=_decode_header(msg.get("In-Reply-To", "")),
+ url="https://mail.google.com/mail/u/0/#inbox",
+ metadata={
+ "message_id": message_id,
+ "imap_uid": uid_text,
+ "imap_uidvalidity": uidvalidity,
+ },
+ )
+ finally:
+ self._close_imap(imap)
+ self._items_synced = synced
def sync_status(self) -> SyncStatus:
return SyncStatus(
diff --git a/src/openjarvis/connectors/imap.py b/src/openjarvis/connectors/imap.py
new file mode 100644
index 00000000..08f43015
--- /dev/null
+++ b/src/openjarvis/connectors/imap.py
@@ -0,0 +1,282 @@
+"""Generic IMAP email connector for any provider."""
+
+from __future__ import annotations
+
+import imaplib
+import ipaddress
+import socket
+import ssl
+from datetime import datetime
+from typing import Iterator, Optional
+
+from openjarvis.connectors._stubs import Document
+from openjarvis.connectors.gmail_imap import (
+ _IMAP_TIMEOUT_SECONDS,
+ _SECURITY_STARTTLS,
+ _SECURITY_TLS,
+ GmailIMAPConnector,
+)
+from openjarvis.core.config import DEFAULT_CONFIG_DIR
+from openjarvis.core.registry import ConnectorRegistry
+
+_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "imap.json")
+
+_PROVIDER_HOSTS = {
+ "gmail.com": "imap.gmail.com",
+ "googlemail.com": "imap.gmail.com",
+ "outlook.com": "outlook.office365.com",
+ "hotmail.com": "outlook.office365.com",
+ "live.com": "outlook.office365.com",
+ "icloud.com": "imap.mail.me.com",
+ "me.com": "imap.mail.me.com",
+ "yahoo.com": "imap.mail.yahoo.com",
+ "fastmail.com": "imap.fastmail.com",
+ "aol.com": "imap.aol.com",
+ "zoho.com": "imap.zoho.com",
+ "gmx.com": "imap.gmx.com",
+}
+
+_BLOCKED_HOSTNAMES = frozenset(
+ {
+ "localhost",
+ "metadata.google.internal",
+ "metadata.google.com",
+ }
+)
+
+
+def _normalize_hostname(host: str) -> str:
+ """Normalize and validate a host without accepting URL-like input."""
+ normalized = host.strip().rstrip(".")
+ if normalized.startswith("[") and normalized.endswith("]"):
+ normalized = normalized[1:-1]
+ if not normalized or any(char.isspace() for char in normalized):
+ raise ValueError("A valid IMAP hostname is required")
+ if any(char in normalized for char in ("/", "\\", "@")) or "://" in normalized:
+ raise ValueError("IMAP host must be a hostname or IP address, not a URL")
+
+ try:
+ return str(ipaddress.ip_address(normalized))
+ except ValueError:
+ pass
+
+ try:
+ ascii_host = normalized.encode("idna").decode("ascii").lower()
+ except UnicodeError as exc:
+ raise ValueError("A valid IMAP hostname is required") from exc
+ labels = ascii_host.split(".")
+ if len(ascii_host) > 253 or any(
+ not label
+ or len(label) > 63
+ or label.startswith("-")
+ or label.endswith("-")
+ or not all(char.isalnum() or char == "-" for char in label)
+ for label in labels
+ ):
+ raise ValueError("A valid IMAP hostname is required")
+ return ascii_host
+
+
+def _is_public_address(address: str) -> bool:
+ """Return whether an address is globally routable unicast."""
+ parsed = ipaddress.ip_address(address)
+ if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped is not None:
+ parsed = parsed.ipv4_mapped
+ # ``is_global`` alone is insufficient: Python classifies multicast ranges
+ # as global even though they are not valid remote unicast endpoints.
+ return parsed.is_global and not any(
+ (
+ parsed.is_multicast,
+ parsed.is_reserved,
+ parsed.is_unspecified,
+ parsed.is_loopback,
+ parsed.is_link_local,
+ parsed.is_private,
+ )
+ )
+
+
+def validate_imap_endpoint(host: str, port: int) -> tuple[str, tuple[str, ...]]:
+ """Resolve a public IMAP endpoint and return addresses safe to pin.
+
+ All resolved addresses must be globally routable. Returning the resolved
+ addresses lets the socket layer pin one of the checked IPs, closing the
+ DNS-rebinding gap between policy validation and connection establishment.
+ """
+ normalized = _normalize_hostname(host)
+ lowered = normalized.lower()
+ if (
+ lowered in _BLOCKED_HOSTNAMES
+ or lowered.endswith(".localhost")
+ or lowered.endswith(".local")
+ or lowered.endswith(".internal")
+ ):
+ raise ValueError("Private or local IMAP hosts are not allowed")
+
+ try:
+ resolved = socket.getaddrinfo(
+ normalized,
+ port,
+ socket.AF_UNSPEC,
+ socket.SOCK_STREAM,
+ )
+ except socket.gaierror as exc:
+ raise ValueError(f"Unable to resolve IMAP host '{normalized}'") from exc
+
+ addresses: list[str] = []
+ for _, _, _, _, sockaddr in resolved:
+ address = sockaddr[0]
+ if not _is_public_address(address):
+ raise ValueError("Private or non-public IMAP addresses are not allowed")
+ if address not in addresses:
+ addresses.append(address)
+ if not addresses:
+ raise ValueError(f"Unable to resolve IMAP host '{normalized}'")
+ return normalized, tuple(addresses)
+
+
+def _create_pinned_socket(address: str, port: int, timeout: float | None):
+ """Create a socket to an already policy-checked IP literal."""
+ if timeout is not None and not timeout:
+ raise ValueError("Non-blocking sockets are not supported")
+ if timeout is None:
+ return socket.create_connection((address, port))
+ return socket.create_connection((address, port), timeout)
+
+
+class _PinnedIMAP4(imaplib.IMAP4):
+ """Plain IMAP transport pinned to a previously validated address."""
+
+ def __init__(
+ self,
+ host: str,
+ port: int,
+ pinned_address: str,
+ *,
+ timeout: float | None = None,
+ ) -> None:
+ self._pinned_address = pinned_address
+ super().__init__(host, port, timeout)
+
+ def _create_socket(self, timeout):
+ return _create_pinned_socket(self._pinned_address, self.port, timeout)
+
+
+class _PinnedIMAP4SSL(imaplib.IMAP4_SSL):
+ """Implicit-TLS IMAP transport with DNS-pinned TCP and hostname TLS."""
+
+ def __init__(
+ self,
+ host: str,
+ port: int,
+ pinned_address: str,
+ *,
+ ssl_context: ssl.SSLContext,
+ timeout: float | None = None,
+ ) -> None:
+ self._pinned_address = pinned_address
+ super().__init__(host, port, ssl_context=ssl_context, timeout=timeout)
+
+ def _create_socket(self, timeout):
+ sock = _create_pinned_socket(self._pinned_address, self.port, timeout)
+ return self.ssl_context.wrap_socket(sock, server_hostname=self.host)
+
+
+def resolve_imap_host(email_address: str) -> str:
+ """Resolve the IMAP host for an email address from its domain."""
+ domain = email_address.rsplit("@", 1)[-1].lower() if "@" in email_address else ""
+ if not domain:
+ return ""
+ return _PROVIDER_HOSTS.get(domain, f"imap.{domain}")
+
+
+class IMAPConnector(GmailIMAPConnector):
+ """Generic IMAP connector for any email provider."""
+
+ connector_id = "imap"
+ display_name = "Email (IMAP)"
+ _default_imap_host = ""
+
+ def __init__(
+ self,
+ email_address: str = "",
+ app_password: str = "",
+ credentials_path: str = "",
+ *,
+ imap_host: str = "",
+ imap_port: int | None = None,
+ imap_security: str = "",
+ max_messages: Optional[int] = None,
+ ) -> None:
+ super().__init__(
+ email_address,
+ app_password,
+ credentials_path or _DEFAULT_CREDENTIALS_PATH,
+ imap_host=imap_host,
+ imap_port=imap_port,
+ imap_security=imap_security,
+ max_messages=max_messages,
+ )
+
+ def auth_url(self) -> str:
+ return ""
+
+ def _host_for_email(self, email_address: str) -> str:
+ return resolve_imap_host(email_address)
+
+ def _make_imap_client(self, host: str, port: int, security: str) -> imaplib.IMAP4:
+ """Connect only to a DNS-pinned, globally routable endpoint."""
+ normalized_host, addresses = validate_imap_endpoint(host, port)
+ tls_context = ssl.create_default_context()
+ last_error: Exception | None = None
+ for address in addresses:
+ try:
+ if security == _SECURITY_TLS:
+ return _PinnedIMAP4SSL(
+ normalized_host,
+ port,
+ address,
+ ssl_context=tls_context,
+ timeout=_IMAP_TIMEOUT_SECONDS,
+ )
+
+ if security != _SECURITY_STARTTLS:
+ raise ValueError("IMAP security must be 'tls' or 'starttls'")
+ imap = _PinnedIMAP4(
+ normalized_host,
+ port,
+ address,
+ timeout=_IMAP_TIMEOUT_SECONDS,
+ )
+ try:
+ status, _ = imap.starttls(ssl_context=tls_context)
+ if status != "OK":
+ raise imaplib.IMAP4.error("IMAP server rejected STARTTLS")
+ return imap
+ except Exception:
+ self._close_imap(imap)
+ raise
+ except (imaplib.IMAP4.error, OSError) as exc:
+ last_error = exc
+
+ if last_error is not None:
+ raise last_error
+ raise OSError(f"Unable to connect to IMAP host '{normalized_host}'")
+
+ def sync(
+ self,
+ *,
+ since: Optional[datetime] = None,
+ cursor: Optional[str] = None,
+ ) -> Iterator[Document]:
+ for doc in super().sync(since=since, cursor=cursor):
+ doc.source = self.connector_id
+ doc.doc_id = doc.doc_id.replace("gmail:", f"{self.connector_id}:", 1)
+ # A generic mailbox must not link every result to Gmail.
+ doc.url = ""
+ yield doc
+
+ def mcp_tools(self) -> list:
+ """Do not expose Gmail-branded live tools for non-Gmail mailboxes."""
+ return []
diff --git a/src/openjarvis/server/connectors_router.py b/src/openjarvis/server/connectors_router.py
index 2e95ec6d..367865b1 100644
--- a/src/openjarvis/server/connectors_router.py
+++ b/src/openjarvis/server/connectors_router.py
@@ -81,6 +81,9 @@ try:
code: Optional[str] = None
email: Optional[str] = None
password: Optional[str] = None
+ host: Optional[str] = None
+ port: Optional[int] = None
+ security: Optional[str] = None
except ImportError:
ConnectRequest = None # type: ignore[assignment,misc]
@@ -409,8 +412,33 @@ def create_connectors_router():
directive = _maybe_oauth_client_pair(connector_id, req)
if directive is not None:
return directive
- if req.code:
- instance.handle_callback(req.code)
+ if (
+ req.email
+ and req.password
+ and hasattr(instance, "connect_with_credentials")
+ ):
+ from starlette.concurrency import run_in_threadpool
+
+ connection_options: Dict[str, Any] = {}
+ # Only the generic IMAP connector accepts a user-selected
+ # network endpoint. Its connector implementation applies
+ # the server-side public-address policy and DNS pinning.
+ if connector_id == "imap":
+ connection_options = {
+ "imap_host": req.host or "",
+ "imap_port": req.port,
+ "imap_security": req.security or "",
+ }
+ await run_in_threadpool(
+ instance.connect_with_credentials,
+ req.email,
+ req.password,
+ **connection_options,
+ )
+ elif req.code:
+ from starlette.concurrency import run_in_threadpool
+
+ await run_in_threadpool(instance.handle_callback, req.code)
elif req.token:
# A credential pasted into the ``token`` field. Connectors
# that accept a pre-existing access token expose ``_token``
diff --git a/tests/connectors/test_imap.py b/tests/connectors/test_imap.py
new file mode 100644
index 00000000..a01cb08b
--- /dev/null
+++ b/tests/connectors/test_imap.py
@@ -0,0 +1,543 @@
+"""Tests for the generic IMAP connector and its network boundary."""
+
+from __future__ import annotations
+
+import json
+import socket
+from imaplib import IMAP4
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from openjarvis.connectors.imap import (
+ IMAPConnector,
+ _create_pinned_socket,
+ resolve_imap_host,
+ validate_imap_endpoint,
+)
+from openjarvis.connectors.oauth import load_tokens
+from openjarvis.core.registry import ConnectorRegistry
+
+
+def _message(
+ *,
+ subject: bytes = b"Test Email",
+ message_id: bytes = b"<[email protected]>",
+) -> bytes:
+ return (
+ b"From: [email protected]\r\n"
+ b"To: [email protected]\r\n"
+ b"Subject: " + subject + b"\r\n"
+ b"Date: Mon, 01 Jan 2024 00:00:00 +0000\r\n"
+ b"Message-ID: " + message_id + b"\r\n\r\n"
+ b"Hello from IMAP test"
+ )
+
+
+def _authenticated_client(uidvalidity: int = 777) -> MagicMock:
+ client = MagicMock()
+ client.login.return_value = ("OK", [])
+ client.select.return_value = ("OK", [])
+ client.response.return_value = ("UIDVALIDITY", [str(uidvalidity).encode("ascii")])
+ client.logout.return_value = ("OK", [])
+ return client
+
+
+def _sync_client(raw_email: bytes | None = None) -> MagicMock:
+ client = _authenticated_client()
+ raw_email = raw_email or _message()
+
+ def uid(command: str, *args):
+ if command == "SEARCH":
+ return "OK", [b"41"]
+ if command == "FETCH":
+ assert args == (b"41", "(RFC822)")
+ return "OK", [(b"41 (RFC822 {1})", raw_email)]
+ raise AssertionError(f"Unexpected UID command: {command}")
+
+ client.uid.side_effect = uid
+ return client
+
+
+def _connect(connector: IMAPConnector, client: MagicMock | None = None) -> MagicMock:
+ client = client or _authenticated_client()
+ with patch.object(connector, "_make_imap_client", return_value=client):
+ connector.handle_callback("[email protected]:pass")
+ return client
+
+
+def test_imap_registered() -> None:
+ ConnectorRegistry.register_value("imap", IMAPConnector)
+ assert ConnectorRegistry.contains("imap")
+ cls = ConnectorRegistry.get("imap")
+ assert cls.connector_id == "imap"
+ assert cls.display_name == "Email (IMAP)"
+
+
+def test_resolve_known_provider_host() -> None:
+ assert resolve_imap_host("[email protected]") == "imap.gmail.com"
+ assert resolve_imap_host("[email protected]") == "outlook.office365.com"
+ assert resolve_imap_host("[email protected]") == "imap.fastmail.com"
+
+
+def test_resolve_unknown_domain_falls_back() -> None:
+ assert resolve_imap_host("[email protected]") == "imap.example.org"
+ assert resolve_imap_host("not-an-email") == ""
+
+
+def test_callback_validates_before_persisting(tmp_path: Path) -> None:
+ creds_path = str(tmp_path / "imap.json")
+ conn = IMAPConnector(credentials_path=creds_path)
+ client = _authenticated_client()
+
+ with patch.object(conn, "_make_imap_client", return_value=client) as make_client:
+ conn.handle_callback("[email protected]:mypassword123")
+
+ make_client.assert_called_once_with("imap.example.org", 993, "tls")
+ client.login.assert_called_once_with("[email protected]", "mypassword123")
+ client.select.assert_called_once_with("INBOX", readonly=True)
+ client.logout.assert_called_once()
+ tokens = load_tokens(creds_path)
+ assert tokens == {
+ "email": "[email protected]",
+ "password": "mypassword123",
+ "imap_host": "imap.example.org",
+ "imap_port": 993,
+ "imap_security": "tls",
+ "validated": True,
+ }
+ assert conn.is_connected() is True
+
+
+def test_failed_login_is_not_persisted_or_connected(tmp_path: Path) -> None:
+ creds_path = str(tmp_path / "imap.json")
+ conn = IMAPConnector(credentials_path=creds_path)
+ client = _authenticated_client()
+ client.login.side_effect = IMAP4.error("bad credentials")
+
+ with (
+ patch.object(conn, "_make_imap_client", return_value=client),
+ pytest.raises(IMAP4.error, match="bad credentials"),
+ ):
+ conn.handle_callback("[email protected]:wrong")
+
+ assert load_tokens(creds_path) is None
+ assert conn.is_connected() is False
+ client.logout.assert_called_once()
+
+
+def test_missing_uidvalidity_is_not_persisted_or_connected(tmp_path: Path) -> None:
+ creds_path = str(tmp_path / "imap.json")
+ conn = IMAPConnector(credentials_path=creds_path)
+ client = _authenticated_client()
+ client.response.return_value = ("UIDVALIDITY", [None])
+
+ with (
+ patch.object(conn, "_make_imap_client", return_value=client),
+ pytest.raises(IMAP4.error, match="UIDVALIDITY"),
+ ):
+ conn.handle_callback("[email protected]:secret")
+
+ assert load_tokens(creds_path) is None
+ assert conn.is_connected() is False
+ client.logout.assert_called_once()
+
+
+def test_unvalidated_legacy_credentials_are_not_connected(tmp_path: Path) -> None:
+ creds_path = tmp_path / "imap.json"
+ creds_path.write_text(
+ json.dumps({"email": "[email protected]", "password": "old"}),
+ encoding="utf-8",
+ )
+ assert IMAPConnector(credentials_path=str(creds_path)).is_connected() is False
+
+
+def test_direct_credentials_do_not_inherit_stored_host_or_validation(
+ tmp_path: Path,
+) -> None:
+ creds_path = tmp_path / "imap.json"
+ creds_path.write_text(
+ json.dumps(
+ {
+ "email": "[email protected]",
+ "password": "old",
+ "imap_host": "old.example.org",
+ "imap_port": 1143,
+ "imap_security": "starttls",
+ "validated": True,
+ }
+ ),
+ encoding="utf-8",
+ )
+ conn = IMAPConnector(
+ email_address="[email protected]",
+ app_password="new",
+ credentials_path=str(creds_path),
+ )
+
+ assert conn.is_connected() is False
+ assert conn._resolve_connection()[2:5] == ("imap.gmail.com", 993, "tls")
+
+
+def test_explicit_starttls_settings_are_validated_and_persisted(
+ tmp_path: Path,
+) -> None:
+ conn = IMAPConnector(credentials_path=str(tmp_path / "imap.json"))
+ client = _authenticated_client()
+
+ with patch.object(conn, "_make_imap_client", return_value=client) as make_client:
+ conn.connect_with_credentials(
+ "secret",
+ imap_host="mail.custom.example",
+ imap_port=1143,
+ imap_security="STARTTLS",
+ )
+
+ make_client.assert_called_once_with("mail.custom.example", 1143, "starttls")
+ tokens = load_tokens(conn._credentials_path)
+ assert tokens is not None
+ assert tokens["imap_host"] == "mail.custom.example"
+ assert tokens["imap_port"] == 1143
+ assert tokens["imap_security"] == "starttls"
+
+
+ ("port", "security", "message"),
+ [
+ (0, "tls", "port"),
+ (65536, "tls", "port"),
+ (993, "plaintext", "security"),
+ ],
+)
+def test_invalid_transport_settings_fail_before_connecting(
+ port: int, security: str, message: str, tmp_path: Path
+) -> None:
+ conn = IMAPConnector(credentials_path=str(tmp_path / "imap.json"))
+ with (
+ patch.object(conn, "_make_imap_client") as make_client,
+ pytest.raises(ValueError, match=message),
+ ):
+ conn.connect_with_credentials(
+ "secret",
+ imap_port=port,
+ imap_security=security,
+ )
+ make_client.assert_not_called()
+
+
+ ("host", "address"),
+ [
+ ("localhost", None),
+ ("metadata.google.internal", None),
+ ("mail.local", None),
+ ("127.0.0.1", "127.0.0.1"),
+ ("10.0.0.8", "10.0.0.8"),
+ ("169.254.169.254", "169.254.169.254"),
+ ("::1", "::1"),
+ ("100.64.0.1", "100.64.0.1"),
+ ("224.0.0.1", "224.0.0.1"),
+ ("239.255.255.250", "239.255.255.250"),
+ ("ff02::1", "ff02::1"),
+ ("::ffff:224.0.0.1", "::ffff:224.0.0.1"),
+ ("0.0.0.0", "0.0.0.0"),
+ ("::", "::"),
+ ("240.0.0.1", "240.0.0.1"),
+ ],
+)
+def test_endpoint_policy_rejects_local_and_nonpublic_hosts(
+ host: str, address: str | None
+) -> None:
+ if address is None:
+ with pytest.raises(ValueError, match="Private|local"):
+ validate_imap_endpoint(host, 993)
+ return
+ family = socket.AF_INET6 if ":" in address else socket.AF_INET
+ sockaddr = (address, 993, 0, 0) if family == socket.AF_INET6 else (address, 993)
+ with (
+ patch(
+ "openjarvis.connectors.imap.socket.getaddrinfo",
+ return_value=[(family, socket.SOCK_STREAM, 6, "", sockaddr)],
+ ),
+ pytest.raises(ValueError, match="non-public"),
+ ):
+ validate_imap_endpoint(host, 993)
+
+
+def test_endpoint_policy_rejects_mixed_public_private_dns() -> None:
+ answers = [
+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 993)),
+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 993)),
+ ]
+ with (
+ patch("openjarvis.connectors.imap.socket.getaddrinfo", return_value=answers),
+ pytest.raises(ValueError, match="non-public"),
+ ):
+ validate_imap_endpoint("mail.example.com", 993)
+
+
+def test_endpoint_policy_returns_only_checked_public_addresses() -> None:
+ answers = [
+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 993)),
+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 993)),
+ (
+ socket.AF_INET6,
+ socket.SOCK_STREAM,
+ 6,
+ "",
+ ("2606:2800:220:1:248:1893:25c8:1946", 993, 0, 0),
+ ),
+ ]
+ with patch("openjarvis.connectors.imap.socket.getaddrinfo", return_value=answers):
+ assert validate_imap_endpoint("MAIL.Example.COM.", 993) == (
+ "mail.example.com",
+ ("93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"),
+ )
+
+
+def test_endpoint_policy_fails_closed_on_dns_error() -> None:
+ with (
+ patch(
+ "openjarvis.connectors.imap.socket.getaddrinfo",
+ side_effect=socket.gaierror("no result"),
+ ),
+ pytest.raises(ValueError, match="Unable to resolve"),
+ ):
+ validate_imap_endpoint("mail.example.com", 993)
+
+
+def test_pinned_socket_connects_to_checked_ip_not_hostname() -> None:
+ sentinel = object()
+ with patch(
+ "openjarvis.connectors.imap.socket.create_connection",
+ return_value=sentinel,
+ ) as create_connection:
+ assert _create_pinned_socket("93.184.216.34", 993, 30) is sentinel
+ create_connection.assert_called_once_with(("93.184.216.34", 993), 30)
+
+
+def test_starttls_is_required_before_login() -> None:
+ conn = IMAPConnector()
+ client = _authenticated_client()
+ client.starttls.return_value = ("OK", [])
+ with (
+ patch(
+ "openjarvis.connectors.imap.validate_imap_endpoint",
+ return_value=("mail.example.com", ("93.184.216.34",)),
+ ),
+ patch("openjarvis.connectors.imap._PinnedIMAP4", return_value=client),
+ ):
+ assert conn._make_imap_client("mail.example.com", 143, "starttls") is client
+ client.starttls.assert_called_once()
+
+
+def test_implicit_tls_uses_checked_ip_and_original_hostname() -> None:
+ conn = IMAPConnector()
+ client = _authenticated_client()
+ context = MagicMock()
+ with (
+ patch(
+ "openjarvis.connectors.imap.validate_imap_endpoint",
+ return_value=("mail.example.com", ("93.184.216.34",)),
+ ),
+ patch(
+ "openjarvis.connectors.imap.ssl.create_default_context",
+ return_value=context,
+ ),
+ patch(
+ "openjarvis.connectors.imap._PinnedIMAP4SSL",
+ return_value=client,
+ ) as pinned_tls,
+ ):
+ assert conn._make_imap_client("mail.example.com", 993, "tls") is client
+
+ pinned_tls.assert_called_once_with(
+ "mail.example.com",
+ 993,
+ "93.184.216.34",
+ ssl_context=context,
+ timeout=30,
+ )
+
+
+def test_sync_uses_uids_and_generic_document_identity(tmp_path: Path) -> None:
+ conn = IMAPConnector(
+ email_address="[email protected]",
+ app_password="secret",
+ credentials_path=str(tmp_path / "imap.json"),
+ )
+ client = _sync_client()
+
+ with patch.object(conn, "_make_imap_client", return_value=client):
+ docs = list(conn.sync())
+
+ assert client.uid.call_args_list[0].args == ("SEARCH", None, "ALL")
+ assert client.uid.call_args_list[1].args == ("FETCH", b"41", "(RFC822)")
+ client.search.assert_not_called()
+ client.fetch.assert_not_called()
+ assert len(docs) == 1
+ assert docs[0].source == "imap"
+ assert docs[0].doc_id == "imap:<[email protected]>"
+ assert docs[0].metadata["imap_uid"] == "41"
+ assert docs[0].metadata["imap_uidvalidity"] == "777"
+ assert docs[0].title == "Test Email"
+ assert docs[0].url == ""
+ client.logout.assert_called_once()
+
+
+def test_missing_message_id_falls_back_to_uid() -> None:
+ conn = IMAPConnector(email_address="[email protected]", app_password="secret")
+ raw = _message(message_id=b"")
+ client = _sync_client(raw)
+ with patch.object(conn, "_make_imap_client", return_value=client):
+ docs = list(conn.sync())
+ assert docs[0].doc_id == "imap:777:41"
+
+
+def test_generator_close_logs_out() -> None:
+ conn = IMAPConnector(
+ email_address="[email protected]",
+ app_password="secret",
+ max_messages=1,
+ )
+ client = _sync_client()
+
+ with patch.object(conn, "_make_imap_client", return_value=client):
+ generator = conn.sync()
+ next(generator)
+ generator.close()
+
+ client.logout.assert_called_once()
+
+
+def test_sync_failed_login_closes_socket_and_marks_disconnected() -> None:
+ conn = IMAPConnector(email_address="[email protected]", app_password="bad")
+ client = _authenticated_client()
+ client.login.side_effect = IMAP4.error("bad credentials")
+
+ with patch.object(conn, "_make_imap_client", return_value=client):
+ assert list(conn.sync()) == []
+
+ assert conn.is_connected() is False
+ client.logout.assert_called_once()
+
+
+def test_generic_imap_does_not_advertise_gmail_tools() -> None:
+ assert IMAPConnector().mcp_tools() == []
+
+
+def test_sync_handles_raw_8bit_headers() -> None:
+ conn = IMAPConnector(email_address="[email protected]", app_password="pass")
+ raw_email = _message(subject=b"caf\xe9 meeting")
+ client = _sync_client(raw_email)
+
+ with patch.object(conn, "_make_imap_client", return_value=client):
+ docs = list(conn.sync())
+
+ assert len(docs) == 1
+ document = docs[0]
+ assert isinstance(document.author, str)
+ assert isinstance(document.title, str)
+ assert all(isinstance(participant, str) for participant in document.participants)
+ json.dumps(
+ {
+ "author": document.author,
+ "title": document.title,
+ "participants": document.participants,
+ **document.metadata,
+ }
+ )
+
+
+def test_sync_reconnects_by_uid_on_dropped_connection() -> None:
+ conn = IMAPConnector(email_address="[email protected]", app_password="pass")
+ first = _authenticated_client()
+
+ def first_uid(command: str, *args):
+ if command == "SEARCH":
+ return "OK", [b"41"]
+ raise OSError("[SSL: BAD_LENGTH] bad length")
+
+ first.uid.side_effect = first_uid
+ second = _authenticated_client()
+
+ def second_uid(command: str, *args):
+ assert command == "FETCH"
+ assert args == (b"41", "(RFC822)")
+ return "OK", [(b"41 (RFC822 {1})", _message())]
+
+ second.uid.side_effect = second_uid
+
+ with patch.object(conn, "_make_imap_client", side_effect=[first, second]):
+ docs = list(conn.sync())
+
+ assert len(docs) == 1
+ assert docs[0].metadata["imap_uid"] == "41"
+ assert docs[0].metadata["imap_uidvalidity"] == "777"
+ first.logout.assert_called_once()
+ second.logout.assert_called_once()
+
+
+def test_sync_stops_if_uidvalidity_changes_during_reconnect() -> None:
+ conn = IMAPConnector(email_address="[email protected]", app_password="pass")
+ first = _authenticated_client(uidvalidity=777)
+
+ def first_uid(command: str, *args):
+ if command == "SEARCH":
+ return "OK", [b"41"]
+ raise OSError("connection dropped")
+
+ first.uid.side_effect = first_uid
+ second = _authenticated_client(uidvalidity=888)
+
+ with patch.object(conn, "_make_imap_client", side_effect=[first, second]):
+ assert list(conn.sync()) == []
+
+ second.uid.assert_not_called()
+ first.logout.assert_called_once()
+ second.logout.assert_called_once()
+
+
+def test_server_rejects_private_custom_host_without_saving_credentials(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ fastapi = pytest.importorskip("fastapi")
+ testclient = pytest.importorskip("fastapi.testclient")
+ from openjarvis.server import connectors_router
+
+ class HermeticIMAPConnector(IMAPConnector):
+ def __init__(self) -> None:
+ super().__init__(credentials_path=str(tmp_path / "imap.json"))
+
+ ConnectorRegistry.register_value("imap", HermeticIMAPConnector)
+ connectors_router._instances.clear()
+ monkeypatch.setattr(
+ "openjarvis.connectors.imap.socket.getaddrinfo",
+ lambda *args, **kwargs: [
+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 1993))
+ ],
+ )
+ app = fastapi.FastAPI()
+ app.include_router(connectors_router.create_connectors_router())
+
+ try:
+ response = testclient.TestClient(app).post(
+ "/v1/connectors/imap/connect",
+ json={
+ "email": "[email protected]",
+ "password": "secret",
+ "host": "mail.attacker.example",
+ "port": 1993,
+ "security": "tls",
+ },
+ )
+ finally:
+ ConnectorRegistry.clear()
+ connectors_router._instances.clear()
+
+ assert response.status_code == 400
+ assert "non-public" in response.json()["detail"]
+ assert not (tmp_path / "imap.json").exists()
diff --git a/tests/connectors/test_outlook.py b/tests/connectors/test_outlook.py
index c6799c97..fa9a4a86 100644
--- a/tests/connectors/test_outlook.py
+++ b/tests/connectors/test_outlook.py
@@ -31,30 +31,40 @@ def test_outlook_auth_url() -> None:
def test_outlook_handle_callback(tmp_path: Path) -> None:
creds_path = str(tmp_path / "outlook.json")
conn = OutlookConnector(credentials_path=creds_path)
- conn.handle_callback("[email protected]:mypassword123")
+ mock_imap = MagicMock()
+ mock_imap.login.return_value = ("OK", [])
+ mock_imap.select.return_value = ("OK", [])
+ mock_imap.response.return_value = ("UIDVALIDITY", [b"777"])
+ with patch.object(conn, "_make_imap_client", return_value=mock_imap):
+ conn.handle_callback("[email protected]:mypassword123")
tokens = load_tokens(creds_path)
assert tokens is not None
assert tokens["email"] == "[email protected]"
assert tokens["password"] == "mypassword123"
+ assert tokens["validated"] is True
def test_outlook_is_connected(tmp_path: Path) -> None:
creds_path = str(tmp_path / "outlook.json")
conn = OutlookConnector(credentials_path=creds_path)
assert conn.is_connected() is False
- conn.handle_callback("[email protected]:pass")
+ mock_imap = MagicMock()
+ mock_imap.login.return_value = ("OK", [])
+ mock_imap.select.return_value = ("OK", [])
+ mock_imap.response.return_value = ("UIDVALIDITY", [b"777"])
+ with patch.object(conn, "_make_imap_client", return_value=mock_imap):
+ conn.handle_callback("[email protected]:pass")
assert conn.is_connected() is True
def test_outlook_sync_source_is_outlook(tmp_path: Path) -> None:
creds_path = str(tmp_path / "outlook.json")
conn = OutlookConnector(credentials_path=creds_path)
- conn.handle_callback("[email protected]:pass")
mock_imap = MagicMock()
mock_imap.login.return_value = ("OK", [])
mock_imap.select.return_value = ("OK", [])
- mock_imap.search.return_value = ("OK", [b"1"])
+ mock_imap.response.return_value = ("UIDVALIDITY", [b"777"])
raw_email = (
b"From: [email protected]\r\n"
@@ -65,15 +75,19 @@ def test_outlook_sync_source_is_outlook(tmp_path: Path) -> None:
b"\r\n"
b"Hello from Outlook test"
)
- mock_imap.fetch.return_value = ("OK", [(b"1", raw_email)])
+ mock_imap.uid.side_effect = [
+ ("OK", [b"1"]),
+ ("OK", [(b"1", raw_email)]),
+ ]
mock_imap.logout.return_value = ("OK", [])
- with patch("openjarvis.connectors.gmail_imap.imaplib") as mock_imaplib:
- mock_imaplib.IMAP4_SSL.return_value = mock_imap
- mock_imaplib.IMAP4 = type(mock_imap)
+ with patch.object(conn, "_make_imap_client", return_value=mock_imap):
+ conn.handle_callback("[email protected]:pass")
docs = list(conn.sync())
assert len(docs) == 1
assert docs[0].source == "outlook"
assert docs[0].doc_id.startswith("outlook:")
assert docs[0].title == "Test Email"
+ assert mock_imap.uid.call_args_list[0].args == ("SEARCH", None, "ALL")
+ assert mock_imap.uid.call_args_list[1].args == ("FETCH", b"1", "(RFC822)")