Email for professionals and teams
feat(search): full-text search with Gmail-style operators on an FTS5 index
Oday Bakkour98100ef93443
diff --git a/CLAUDE.md b/CLAUDE.mdindex 5a1b6e7..3c63691 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -79,6 +79,10 @@ Schema lives in one file: `src/db/schema/index.ts` (21 tables). Migrations are gThe setup path only ever initializes an empty database — it refuses to touch one that already has tables.+### Search is an FTS5 index kept by triggers++`messages_fts` (migration 0030) is an external-content FTS5 table over `messages`; three triggers in the same migration keep it in sync on insert, update and delete, so no application code touches the index. `buildSearchConditions` in `src/lib/search/conditions.ts` turns the Gmail-style grammar (`src/lib/search/query-utils.ts`) into a `rowid IN (SELECT rowid FROM messages_fts WHERE messages_fts MATCH ?)` predicate plus plain column filters. Two consequences: the bootstrap SQL in `src/lib/setup/migration.ts` is split with `splitSqlStatements`, which keeps trigger bodies whole; and the backup coverage check skips `messages_fts%`, since the shadow tables are derived and repopulate on restore. `wrangler d1 export` does not work on databases with virtual tables; the app's own JSON backup is unaffected.+### Access controlTwo independent auth surfaces:diff --git a/docs/api.md b/docs/api.mdindex eeccbd8..144162c 100644--- a/docs/api.md+++ b/docs/api.md@@ -57,6 +57,22 @@ Messages composed in Mailflare are sent as HTML with a plain-text alternative deThe dashboard composer accepts up to 10 attachments, with a 10 MB limit per file and a 20 MB combined limit. Attachment metadata is stored in D1 and file content is stored in R2. Downloads require access to the mailbox containing the message.+## Searching++`GET /api/messages?q=...` (session) and `GET /api/v1/messages?q=...` (API key, `read` scope) accept the same query grammar, backed by an FTS5 index over subject, sender, recipients and body:++| Syntax | Meaning |+|---|---|+| `invoice` | prefix match anywhere (`inv` finds "invoice") |+| `"private window"` | exact phrase |+| `-word` | exclude |+| `from:maya`, `to:sam`, `subject:report` | restrict a term to one field (`to:` covers Cc) |+| `has:attachment` | at least one non-inline attachment |+| `is:unread`, `is:read`, `is:starred` | flags |+| `after:2026-09-01`, `before:2026-09-30` | date bounds (UTC, `before` exclusive) |++Terms combine with AND. Admins can check or rebuild the index with `GET` / `POST /api/admin/search-index`; triggers keep it current, so a rebuild is only needed after restoring a backup made before the index existed.+## Real-time updatesMailflare uses a Durable Object WebSocket hub to notify connected users after an inbound message is stored. Mailbox owners, the domain administrator, and delegated users receive events for mailboxes they can access.diff --git a/drizzle/migrations/0030_add_message_search_index.sql b/drizzle/migrations/0030_add_message_search_index.sqlnew file mode 100644index 0000000..904bcf5--- /dev/null+++ b/drizzle/migrations/0030_add_message_search_index.sql@@ -0,0 +1,29 @@+-- Full-text search over messages. An external-content FTS5 index reads its+-- text from the messages table, so bodies are not stored twice; triggers keep+-- it in step with every insert, update and delete, whichever code path writes.+CREATE VIRTUAL TABLE `messages_fts` USING fts5(+ `subject`,+ `from_addr`,+ `to_addr`,+ `cc_addr`,+ `text_body`,+ `html_body`,+ content='messages',+ content_rowid='rowid',+ tokenize='unicode61 remove_diacritics 2'+);--> statement-breakpoint+CREATE TRIGGER `messages_fts_ai` AFTER INSERT ON `messages` BEGIN+ INSERT INTO `messages_fts`(`rowid`, `subject`, `from_addr`, `to_addr`, `cc_addr`, `text_body`, `html_body`)+ VALUES (new.`rowid`, new.`subject`, new.`from_addr`, new.`to_addr`, new.`cc_addr`, new.`text_body`, new.`html_body`);+END;--> statement-breakpoint+CREATE TRIGGER `messages_fts_ad` AFTER DELETE ON `messages` BEGIN+ INSERT INTO `messages_fts`(`messages_fts`, `rowid`, `subject`, `from_addr`, `to_addr`, `cc_addr`, `text_body`, `html_body`)+ VALUES ('delete', old.`rowid`, old.`subject`, old.`from_addr`, old.`to_addr`, old.`cc_addr`, old.`text_body`, old.`html_body`);+END;--> statement-breakpoint+CREATE TRIGGER `messages_fts_au` AFTER UPDATE OF `subject`, `from_addr`, `to_addr`, `cc_addr`, `text_body`, `html_body` ON `messages` BEGIN+ INSERT INTO `messages_fts`(`messages_fts`, `rowid`, `subject`, `from_addr`, `to_addr`, `cc_addr`, `text_body`, `html_body`)+ VALUES ('delete', old.`rowid`, old.`subject`, old.`from_addr`, old.`to_addr`, old.`cc_addr`, old.`text_body`, old.`html_body`);+ INSERT INTO `messages_fts`(`rowid`, `subject`, `from_addr`, `to_addr`, `cc_addr`, `text_body`, `html_body`)+ VALUES (new.`rowid`, new.`subject`, new.`from_addr`, new.`to_addr`, new.`cc_addr`, new.`text_body`, new.`html_body`);+END;--> statement-breakpoint+INSERT INTO `messages_fts`(`messages_fts`) VALUES ('rebuild');diff --git a/drizzle/migrations/meta/_journal.json b/drizzle/migrations/meta/_journal.jsonindex 0f60979..514ad9a 100644--- a/drizzle/migrations/meta/_journal.json+++ b/drizzle/migrations/meta/_journal.json@@ -204,6 +204,20 @@"when": 1788850800000,"tag": "0028_add_keyboard_shortcuts_setting","breakpoints": true+ },+ {+ "idx": 29,+ "version": "6",+ "when": 1788850800001,+ "tag": "0029_add_spam_protection",+ "breakpoints": true+ },+ {+ "idx": 30,+ "version": "6",+ "when": 1788850800002,+ "tag": "0030_add_message_search_index",+ "breakpoints": true}]}diff --git a/src/app/api/admin/search-index/route.ts b/src/app/api/admin/search-index/route.tsnew file mode 100644index 0000000..c718167--- /dev/null+++ b/src/app/api/admin/search-index/route.ts@@ -0,0 +1,32 @@+import { NextResponse } from "next/server";+import { assertAdmin } from "@/lib/auth/admin";+import { requireSessionUser } from "@/lib/api/auth";+import { getEnv } from "@/lib/cloudflare";+import { getSearchIndexStatus, rebuildSearchIndex } from "@/lib/search/index-admin";++/** Row counts for the index versus the messages table, to spot drift. */+export async function GET(request: Request) {+ const env = getEnv();+ const auth = await requireSessionUser(env, request);+ if (auth.error) return auth.error;+ try {+ assertAdmin(auth.user);+ } catch {+ return NextResponse.json({ error: "Forbidden" }, { status: 403 });+ }+ return NextResponse.json(await getSearchIndexStatus(env));+}++/** Re-index every message from the messages table. Safe to run at any time. */+export async function POST(request: Request) {+ const env = getEnv();+ const auth = await requireSessionUser(env, request);+ if (auth.error) return auth.error;+ try {+ assertAdmin(auth.user);+ } catch {+ return NextResponse.json({ error: "Forbidden" }, { status: 403 });+ }+ await rebuildSearchIndex(env);+ return NextResponse.json(await getSearchIndexStatus(env));+}diff --git a/src/app/api/messages/route.ts b/src/app/api/messages/route.tsindex 7025385..963b995 100644--- a/src/app/api/messages/route.ts+++ b/src/app/api/messages/route.ts@@ -1,5 +1,5 @@import { NextResponse } from "next/server";-import { eq, desc, and, like, or, count, countDistinct, isNull, isNotNull, inArray, lte, gt, max, notInArray, sql, sum } from "drizzle-orm";+import { eq, desc, and, or, count, countDistinct, isNull, isNotNull, inArray, lte, gt, max, notInArray, sql, sum } from "drizzle-orm";import type { SQL } from "drizzle-orm";import { getEnv } from "@/lib/cloudflare";import { getCurrentUser } from "@/lib/auth/cookies";@@ -9,6 +9,7 @@ import { getContactDisplayNameMap } from "@/lib/contacts/service";import { getFirstEmailAddressEntry, normalizeEmailAddress } from "@/lib/email/address";import { buildSnippet } from "@/lib/email/parse";import { getMailboxAccessLevel, listAccessibleMailboxes } from "@/lib/mailboxes/access";+import { buildSearchConditions } from "@/lib/search/conditions";export async function GET(request: Request) {const env = getEnv();@@ -75,18 +76,10 @@ export async function GET(request: Request) {if (read === "unread") {conditions.push(eq(messages.read, false));}- if (query) {- const pattern = `%${query}%`;- const queryCondition = or(- like(messages.fromAddr, pattern),- like(messages.toAddr, pattern),- like(messages.subject, pattern),- like(messages.snippet, pattern),- );- if (queryCondition) conditions.push(queryCondition);- }- if (title) {- conditions.push(like(messages.subject, `%${title}%`));+ if (query || title) {+ // Operators (from:, has:attachment, before:) and free text go through the+ // full-text index; `title` is the legacy subject filter and is folded in.+ conditions.push(...buildSearchConditions(title ? `${query ?? ""} subject:"${title}"` : query ?? ""));}const where = and(...conditions);// Messages that were never threaded (older rows, drafts) stand alone.diff --git a/src/app/api/v1/messages/route.ts b/src/app/api/v1/messages/route.tsindex 17b8504..08b8a13 100644--- a/src/app/api/v1/messages/route.ts+++ b/src/app/api/v1/messages/route.ts@@ -6,6 +6,7 @@ import { authenticateApiKey, requireScope } from "@/lib/api/auth";import { getDb } from "@/db";import { messages, users } from "@/db/schema";import { getMailboxAccessLevel, listAccessibleMailboxIds } from "@/lib/mailboxes/access";+import { buildSearchConditions } from "@/lib/search/conditions";export async function GET(request: Request) {const env = getEnv();@@ -17,6 +18,7 @@ export async function GET(request: Request) {const url = new URL(request.url);const mailboxId = url.searchParams.get("mailboxId");const direction = url.searchParams.get("direction");+ const query = url.searchParams.get("q")?.trim();const limit = Math.min(Number(url.searchParams.get("limit") ?? 50), 100);const db = getDb(env);@@ -42,6 +44,7 @@ export async function GET(request: Request) {if (direction === "inbound" || direction === "outbound") {conditions.push(eq(messages.direction, direction));}+ if (query) conditions.push(...buildSearchConditions(query));const rows = await db.select()diff --git a/src/components/mail-search/mail-search-context.tsx b/src/components/mail-search/mail-search-context.tsxindex 41672fd..ee9fbd2 100644--- a/src/components/mail-search/mail-search-context.tsx+++ b/src/components/mail-search/mail-search-context.tsx@@ -1,6 +1,6 @@"use client";-import { createContext, useContext, useState } from "react";+import { createContext, useContext, useEffect, useState } from "react";import type { ReactNode } from "react";import type { MailSearchContextValue } from "./types";@@ -12,11 +12,23 @@ export function useMailSearch() {return ctx;}+const SEARCH_DEBOUNCE_MS = 250;+export function MailSearchProvider({ children }: { children: ReactNode }) {+ // `input` follows every keystroke; `query` is what lists fetch with, and+ // trails it slightly so a typed word costs one request instead of one per letter.+ const [input, setInput] = useState("");const [query, setQuery] = useState("");+ useEffect(() => {+ // Clearing the box should empty the list at once; typing waits for a pause.+ const delay = input.trim() ? SEARCH_DEBOUNCE_MS : 0;+ const timer = setTimeout(() => setQuery(input.trim() ? input : ""), delay);+ return () => clearTimeout(timer);+ }, [input]);+return (- <MailSearchContext.Provider value={{ query, setQuery }}>+ <MailSearchContext.Provider value={{ query, input, setQuery: setInput }}>{children}</MailSearchContext.Provider>);diff --git a/src/components/mail-search/mail-search-input.tsx b/src/components/mail-search/mail-search-input.tsxindex ef22f77..39d5be9 100644--- a/src/components/mail-search/mail-search-input.tsx+++ b/src/components/mail-search/mail-search-input.tsx@@ -6,7 +6,7 @@ import { useMailSearch } from "./mail-search-context";import { useShortcuts } from "@/components/shortcuts";export function MailSearchInput() {- const { query, setQuery } = useMailSearch();+ const { input: query, setQuery } = useMailSearch();const { openCommandPalette, shortcutsEnabled, shortcutsPreferenceLoading } = useShortcuts();const showShortcutHints = shortcutsEnabled && !shortcutsPreferenceLoading;diff --git a/src/components/mail-search/types.d.ts b/src/components/mail-search/types.d.tsindex 73de578..1f34208 100644--- a/src/components/mail-search/types.d.ts+++ b/src/components/mail-search/types.d.ts@@ -1,4 +1,7 @@export type MailSearchContextValue = {+ /** Debounced value lists fetch with. */query: string;+ /** Live value of the search box. */+ input: string;setQuery: (query: string) => void;};diff --git a/src/hooks/utils.ts b/src/hooks/utils.tsindex c87be29..f54e82e 100644--- a/src/hooks/utils.ts+++ b/src/hooks/utils.ts@@ -1,30 +1,21 @@import { authFetch } from "@/lib/auth/client";+import { parseSearchQuery } from "@/lib/search/query-utils";import type { MessageFilterOptions, MessageFolder } from "./types";import type { MessageCounts, MessageListResponse } from "./types";export const MESSAGE_POLL_INTERVAL_MS = 15_000;+/**+ * The search string goes to the server whole; operators are parsed there+ * against the full-text index. Only the read state is lifted out here so the+ * unread toggle and `is:unread` share one `read` parameter.+ */export function parseMessageSearchQuery(query: string): MessageFilterOptions {- let remaining = query;+ const parsed = parseSearchQuery(query);const filters: MessageFilterOptions = {};- const titleMatch = remaining.match(/\btitle:"([^"]+)"/i) ?? remaining.match(/\btitle:([^\s]+)/i);-- if (titleMatch?.[1]) {- filters.title = titleMatch[1].trim();- remaining = remaining.replace(titleMatch[0], " ");- }-- if (/(^|\s):unread(\s|$)/i.test(remaining)) {- filters.read = "unread";- remaining = remaining.replace(/(^|\s):unread(?=\s|$)/gi, " ");- } else if (/(^|\s):read(\s|$)/i.test(remaining)) {- filters.read = "read";- remaining = remaining.replace(/(^|\s):read(?=\s|$)/gi, " ");- }-- const textQuery = remaining.replace(/\s+/g, " ").trim();- if (textQuery) filters.query = textQuery;-+ if (parsed.read) filters.read = parsed.read;+ const remaining = query.replace(/(^|\s)(is:(un)?read|:(un)?read)(?=\s|$)/gi, " ").replace(/\s+/g, " ").trim();+ if (remaining) filters.query = remaining;return filters;}diff --git a/src/lib/backups/export.ts b/src/lib/backups/export.tsindex e97ebff..02cc3d2 100644--- a/src/lib/backups/export.ts+++ b/src/lib/backups/export.ts@@ -1,7 +1,7 @@import type { DatabaseBackupDocument, DatabaseBackupTable, DatabaseRecord } from "./types";import { mergeLegacyMessageBodies } from "./utils";-const BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings", "email_templates", "calendar_events", "auto_reply_deliveries", "spam_token_stats", "spam_reputation", "spam_feedback"];+const BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings", "email_templates", "calendar_events", "auto_reply_deliveries", "spam_token_stats", "spam_reputation", "spam_feedback", "mailbox_aliases"];/*** Tables every backup document must contain. Tables added to BACKUP_TABLES* after the format shipped are absent from older documents, so they stay@@ -17,7 +17,11 @@ export function getBackupConfigurationStatus(_env?: CloudflareEnv) {/*** Tables D1 manages itself, which are intentionally absent from BACKUP_TABLES.*/-const INTERNAL_TABLE_PATTERNS = ["sqlite_%", "_cf%"];+const INTERNAL_TABLE_PATTERNS = ["sqlite_%", "_cf%", "messages_fts%"];+/**+ * The search index is derived data: its triggers repopulate it as messages are+ * restored, so it is neither exported nor part of the coverage check.+ */const INTERNAL_TABLES = ["d1_migrations"];/**diff --git a/src/lib/backups/types.d.ts b/src/lib/backups/types.d.tsindex ac6d5bf..2518b27 100644--- a/src/lib/backups/types.d.ts+++ b/src/lib/backups/types.d.ts@@ -1,5 +1,5 @@export type BackupScheduleType = "daily" | "weekly" | "monthly";-export type DatabaseBackupTable = "users" | "domains" | "mailboxes" | "mailbox_access" | "contacts" | "folders" | "api_keys" | "messages" | "message_attachments" | "outbound_jobs" | "routing_rules" | "webhooks" | "webhook_deliveries" | "sessions" | "audit_logs" | "backup_settings" | "backups" | "app_settings" | "license_settings" | "email_templates" | "calendar_events" | "auto_reply_deliveries" | "spam_token_stats" | "spam_reputation" | "spam_feedback";+export type DatabaseBackupTable = "users" | "domains" | "mailboxes" | "mailbox_access" | "contacts" | "folders" | "api_keys" | "messages" | "message_attachments" | "outbound_jobs" | "routing_rules" | "webhooks" | "webhook_deliveries" | "sessions" | "audit_logs" | "backup_settings" | "backups" | "app_settings" | "license_settings" | "email_templates" | "calendar_events" | "auto_reply_deliveries" | "spam_token_stats" | "spam_reputation" | "spam_feedback" | "mailbox_aliases";export type DatabaseRecord = Record<string, string | number | null>;export type DatabaseBackupDocument = { format: "mailflare-database-backup"; version: 1; createdAt: string; tables: Record<DatabaseBackupTable, DatabaseRecord[]>; };diff --git a/src/lib/search/conditions.ts b/src/lib/search/conditions.tsnew file mode 100644index 0000000..8eca20c--- /dev/null+++ b/src/lib/search/conditions.ts@@ -0,0 +1,33 @@+import { and, eq, exists, gte, lt, sql } from "drizzle-orm";+import type { SQL } from "drizzle-orm";+import { messageAttachments, messages } from "@/db/schema";+import { buildFtsMatch, parseSearchQuery } from "./query-utils";++/**+ * Translate a search string into WHERE conditions on `messages`. Text goes to+ * the FTS5 index by rowid; everything else is a plain column predicate, so the+ * caller ANDs these with its own scope (mailbox access, folder, status).+ */+export function buildSearchConditions(raw: string): SQL[] {+ const parsed = parseSearchQuery(raw);+ const conditions: SQL[] = [];++ const match = buildFtsMatch(parsed);+ if (match) {+ conditions.push(sql`${messages}.rowid IN (SELECT rowid FROM messages_fts WHERE messages_fts MATCH ${match})`);+ }+ if (parsed.hasAttachment) {+ conditions.push(+ exists(+ sql`(SELECT 1 FROM ${messageAttachments} WHERE ${messageAttachments.messageId} = ${messages.id} AND ${messageAttachments.disposition} = 'attachment')`,+ ),+ );+ }+ if (parsed.read === "read") conditions.push(eq(messages.read, true));+ if (parsed.read === "unread") conditions.push(eq(messages.read, false));+ if (parsed.starred) conditions.push(eq(messages.starred, true));+ if (parsed.after) conditions.push(gte(messages.createdAt, parsed.after));+ if (parsed.before) conditions.push(lt(messages.createdAt, parsed.before));++ return conditions.length ? [and(...conditions)!] : [];+}diff --git a/src/lib/search/index-admin.ts b/src/lib/search/index-admin.tsnew file mode 100644index 0000000..3025c59--- /dev/null+++ b/src/lib/search/index-admin.ts@@ -0,0 +1,16 @@+/**+ * Maintenance for the FTS5 index. Triggers keep it current, so a rebuild is+ * only needed after a restore from a backup that predates the index, or if the+ * counts below ever disagree.+ */+export async function rebuildSearchIndex(env: CloudflareEnv): Promise<void> {+ await env.DB.prepare("INSERT INTO messages_fts(messages_fts) VALUES ('rebuild')").run();+}++export async function getSearchIndexStatus(env: CloudflareEnv): Promise<{ indexed: number; messages: number }> {+ const [indexed, total] = await env.DB.batch<{ n: number }>([+ env.DB.prepare("SELECT count(*) AS n FROM messages_fts"),+ env.DB.prepare("SELECT count(*) AS n FROM messages"),+ ]);+ return { indexed: indexed.results[0]?.n ?? 0, messages: total.results[0]?.n ?? 0 };+}diff --git a/src/lib/search/query-utils.ts b/src/lib/search/query-utils.tsnew file mode 100644index 0000000..70de828--- /dev/null+++ b/src/lib/search/query-utils.ts@@ -0,0 +1,140 @@+import type { ParsedSearchQuery, SearchToken } from "./types";++/**+ * Gmail-style search grammar shared by the search box and the API:+ *+ * from:maya to:"sam okoro" subject:invoice has:attachment is:unread+ * is:starred after:2026-09-01 before:2026-09-30 "exact phrase" -excluded word+ *+ * The older `title:` and bare `:unread` / `:read` forms keep working.+ */+const OPERATOR_RE = /(?:^|\s)(from|to|subject|title|has|is|after|before|newer|older):(?:"([^"]*)"|(\S+))/gi;++export function parseSearchQuery(raw: string): ParsedSearchQuery {+ const parsed: ParsedSearchQuery = { text: "", needsFullText: false };+ let rest = raw ?? "";++ rest = rest.replace(OPERATOR_RE, (_match, key: string, quoted: string | undefined, bare: string | undefined) => {+ const value = (quoted ?? bare ?? "").trim();+ switch (key.toLowerCase()) {+ case "from":+ parsed.from = value;+ break;+ case "to":+ parsed.to = value;+ break;+ case "subject":+ case "title":+ parsed.subject = value;+ break;+ case "has":+ if (/^attachments?$/i.test(value)) parsed.hasAttachment = true;+ break;+ case "is":+ if (/^unread$/i.test(value)) parsed.read = "unread";+ else if (/^read$/i.test(value)) parsed.read = "read";+ else if (/^starred$/i.test(value)) parsed.starred = true;+ break;+ case "after":+ case "newer":+ parsed.after = parseDate(value) ?? parsed.after;+ break;+ case "before":+ case "older":+ parsed.before = parseDate(value) ?? parsed.before;+ break;+ }+ return " ";+ });++ if (/(^|\s):unread(?=\s|$)/i.test(rest)) {+ parsed.read = "unread";+ rest = rest.replace(/(^|\s):unread(?=\s|$)/gi, " ");+ } else if (/(^|\s):read(?=\s|$)/i.test(rest)) {+ parsed.read = "read";+ rest = rest.replace(/(^|\s):read(?=\s|$)/gi, " ");+ }++ parsed.text = rest.replace(/\s+/g, " ").trim();+ parsed.needsFullText = !!(parsed.text || parsed.from || parsed.to || parsed.subject);+ return parsed;+}++function parseDate(value: string): Date | undefined {+ const match = value.match(/^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$/);+ if (!match) return undefined;+ const date = new Date(Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3])));+ return Number.isNaN(date.getTime()) ? undefined : date;+}++/** Split free text into phrases ("..."), negations (-word) and plain terms. */+export function tokenizeSearchText(text: string): SearchToken[] {+ const tokens: SearchToken[] = [];+ const re = /(-?)"([^"]+)"|(-?)(\S+)/g;+ let match: RegExpExecArray | null;+ while ((match = re.exec(text))) {+ if (match[2] !== undefined) {+ const term = match[2].trim();+ if (term) tokens.push({ term, negate: match[1] === "-", phrase: true });+ } else {+ const term = match[4].replace(/^-+/, "").trim();+ if (term) tokens.push({ term, negate: match[3] === "-", phrase: false });+ }+ }+ return tokens;+}++/** FTS5 string literal: double quotes doubled, so user input can never alter the query grammar. */+function ftsString(value: string): string {+ return `"${value.replace(/"/g, '""')}"`;+}++/**+ * One term. Phrases match whole words; plain terms match by prefix so "inv"+ * finds "invoice". Punctuation FTS5 treats as separators is normalised to+ * spaces first, so "[email protected]" becomes the phrase "maya acme test".+ */+function ftsTerm(term: string, phrase: boolean): string {+ const cleaned = term.replace(/[^\p{L}\p{N}\s]+/gu, " ").replace(/\s+/g, " ").trim();+ if (!cleaned) return "";+ if (phrase || cleaned.includes(" ")) {+ // A multi-word prefix query needs the last word starred, which FTS5 only+ // allows inside a phrase as `"a b" *`; keep it simple and require whole words.+ return ftsString(cleaned);+ }+ return `${ftsString(cleaned)}*`;+}++/**+ * Build the FTS5 MATCH expression. Column filters restrict a term to one+ * column; free text is ANDed across the whole row. Returns null when there is+ * nothing to match, so callers skip the index entirely.+ */+export function buildFtsMatch(parsed: ParsedSearchQuery): string | null {+ const parts: string[] = [];++ for (const token of tokenizeSearchText(parsed.text)) {+ const expr = ftsTerm(token.term, token.phrase);+ if (!expr) continue;+ parts.push(token.negate ? `NOT ${expr}` : expr);+ }+ const column = (name: string, value: string | undefined) => {+ if (!value) return;+ const terms = tokenizeSearchText(value)+ .map((token) => ftsTerm(token.term, token.phrase))+ .filter(Boolean);+ if (terms.length === 0) return;+ parts.push(`${name} : (${terms.join(" AND ")})`);+ };+ column("subject", parsed.subject);+ column("from_addr", parsed.from);+ column("{to_addr cc_addr}", parsed.to);++ if (parts.length === 0) return null;+ // FTS5 has no unary NOT: "a NOT b" is binary. A leading negation is turned+ // into "match anything" minus the term.+ const positives = parts.filter((part) => !part.startsWith("NOT "));+ const negatives = parts.filter((part) => part.startsWith("NOT ")).map((part) => part.slice(4));+ const base = positives.length > 0 ? positives.join(" AND ") : `${ftsString("")}*`;+ return negatives.length > 0 ? `(${base}) NOT (${negatives.join(" OR ")})` : base;+}diff --git a/src/lib/search/types.d.ts b/src/lib/search/types.d.tsnew file mode 100644index 0000000..17b5818--- /dev/null+++ b/src/lib/search/types.d.ts@@ -0,0 +1,18 @@+export type ParsedSearchQuery = {+ /** Free text after operators are removed; matched against every indexed column. */+ text: string;+ from?: string;+ to?: string;+ subject?: string;+ hasAttachment?: boolean;+ read?: "read" | "unread";+ starred?: boolean;+ /** Inclusive lower bound on the message date, from `after:YYYY-MM-DD`. */+ after?: Date;+ /** Exclusive upper bound on the message date, from `before:YYYY-MM-DD`. */+ before?: Date;+ /** True when anything in the query needs the full-text index. */+ needsFullText: boolean;+};++export type SearchToken = { term: string; negate: boolean; phrase: boolean };diff --git a/src/lib/setup/migration.ts b/src/lib/setup/migration.tsindex d91ec72..43d7d8f 100644--- a/src/lib/setup/migration.ts+++ b/src/lib/setup/migration.ts@@ -29,6 +29,7 @@ const MIGRATION_NAMES = ["0027_add_domain_sending_intent.sql","0028_add_keyboard_shortcuts_setting.sql","0029_add_spam_protection.sql",+ "0030_add_message_search_index.sql",];const INITIAL_SCHEMA_SQL = `@@ -61,6 +62,10 @@ CREATE TABLE IF NOT EXISTS messages (id text PRIMARY KEY NOT NULL, user_id textCREATE INDEX IF NOT EXISTS messages_user_created_idx ON messages(user_id, created_at);CREATE INDEX IF NOT EXISTS messages_mailbox_idx ON messages(mailbox_id);CREATE INDEX IF NOT EXISTS messages_folder_idx ON messages(folder_id);+CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(subject, from_addr, to_addr, cc_addr, text_body, html_body, content='messages', content_rowid='rowid', tokenize='unicode61 remove_diacritics 2');+CREATE TRIGGER IF NOT EXISTS messages_fts_ai AFTER INSERT ON messages BEGIN INSERT INTO messages_fts(rowid, subject, from_addr, to_addr, cc_addr, text_body, html_body) VALUES (new.rowid, new.subject, new.from_addr, new.to_addr, new.cc_addr, new.text_body, new.html_body); END;+CREATE TRIGGER IF NOT EXISTS messages_fts_ad AFTER DELETE ON messages BEGIN INSERT INTO messages_fts(messages_fts, rowid, subject, from_addr, to_addr, cc_addr, text_body, html_body) VALUES ('delete', old.rowid, old.subject, old.from_addr, old.to_addr, old.cc_addr, old.text_body, old.html_body); END;+CREATE TRIGGER IF NOT EXISTS messages_fts_au AFTER UPDATE OF subject, from_addr, to_addr, cc_addr, text_body, html_body ON messages BEGIN INSERT INTO messages_fts(messages_fts, rowid, subject, from_addr, to_addr, cc_addr, text_body, html_body) VALUES ('delete', old.rowid, old.subject, old.from_addr, old.to_addr, old.cc_addr, old.text_body, old.html_body); INSERT INTO messages_fts(rowid, subject, from_addr, to_addr, cc_addr, text_body, html_body) VALUES (new.rowid, new.subject, new.from_addr, new.to_addr, new.cc_addr, new.text_body, new.html_body); END;CREATE INDEX IF NOT EXISTS messages_thread_idx ON messages(mailbox_id, thread_id);CREATE INDEX IF NOT EXISTS messages_provider_message_idx ON messages(mailbox_id, provider_message_id);CREATE INDEX IF NOT EXISTS messages_raw_r2_key_idx ON messages(raw_r2_key);@@ -116,14 +121,32 @@ export async function migrateCleanDatabase(db: D1Database): Promise<boolean> {);}- const schemaStatements = INITIAL_SCHEMA_SQL- .split(";")- .map((statement) => statement.trim())- .filter(Boolean)- .map((statement) => db.prepare(statement));+ const schemaStatements = splitSqlStatements(INITIAL_SCHEMA_SQL).map((statement) => db.prepare(statement));const migrationStatements = MIGRATION_NAMES.map((name) =>db.prepare("INSERT OR IGNORE INTO d1_migrations (name) VALUES (?)").bind(name),);await db.batch([...schemaStatements, ...migrationStatements]);return true;}++/**+ * Split the bootstrap SQL into statements. A plain split on ";" would cut a+ * trigger body apart, so pieces are joined back together until every BEGIN+ * has its END.+ */+export function splitSqlStatements(sqlText: string): string[] {+ const statements: string[] = [];+ let current = "";+ for (const piece of sqlText.split(";")) {+ current = current ? `${current};${piece}` : piece;+ const opened = (current.match(/\bBEGIN\b/gi) ?? []).length;+ const closed = (current.match(/\bEND\b/gi) ?? []).length;+ if (opened > closed) continue;+ const statement = current.trim();+ if (statement) statements.push(statement);+ current = "";+ }+ const rest = current.trim();+ if (rest) statements.push(rest);+ return statements;+}