Skip to content

Email for professionals and teams

feat(search): full-text search with Gmail-style operators on an FTS5 index

Oday Bakkour98100ef93443
diff --git a/CLAUDE.md b/CLAUDE.md
index 5a1b6e7..3c63691 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -79,6 +79,10 @@ Schema lives in one file: `src/db/schema/index.ts` (21 tables). Migrations are g
The setup path only ever initializes an empty database — it refuses to touch one that already has tables.
+### Search is an FTS5 index kept by triggers
+
+`messages_fts` (migration 0030) is an external-content FTS5 table over `messages`; three triggers in the same migration keep it in sync on insert, update and delete, so no application code touches the index. `buildSearchConditions` in `src/lib/search/conditions.ts` turns the Gmail-style grammar (`src/lib/search/query-utils.ts`) into a `rowid IN (SELECT rowid FROM messages_fts WHERE messages_fts MATCH ?)` predicate plus plain column filters. Two consequences: the bootstrap SQL in `src/lib/setup/migration.ts` is split with `splitSqlStatements`, which keeps trigger bodies whole; and the backup coverage check skips `messages_fts%`, since the shadow tables are derived and repopulate on restore. `wrangler d1 export` does not work on databases with virtual tables; the app's own JSON backup is unaffected.
+
### Access control
Two independent auth surfaces:
diff --git a/docs/api.md b/docs/api.md
index eeccbd8..144162c 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -57,6 +57,22 @@ Messages composed in Mailflare are sent as HTML with a plain-text alternative de
The dashboard composer accepts up to 10 attachments, with a 10 MB limit per file and a 20 MB combined limit. Attachment metadata is stored in D1 and file content is stored in R2. Downloads require access to the mailbox containing the message.
+## Searching
+
+`GET /api/messages?q=...` (session) and `GET /api/v1/messages?q=...` (API key, `read` scope) accept the same query grammar, backed by an FTS5 index over subject, sender, recipients and body:
+
+| Syntax | Meaning |
+|---|---|
+| `invoice` | prefix match anywhere (`inv` finds "invoice") |
+| `"private window"` | exact phrase |
+| `-word` | exclude |
+| `from:maya`, `to:sam`, `subject:report` | restrict a term to one field (`to:` covers Cc) |
+| `has:attachment` | at least one non-inline attachment |
+| `is:unread`, `is:read`, `is:starred` | flags |
+| `after:2026-09-01`, `before:2026-09-30` | date bounds (UTC, `before` exclusive) |
+
+Terms combine with AND. Admins can check or rebuild the index with `GET` / `POST /api/admin/search-index`; triggers keep it current, so a rebuild is only needed after restoring a backup made before the index existed.
+
## Real-time updates
Mailflare uses a Durable Object WebSocket hub to notify connected users after an inbound message is stored. Mailbox owners, the domain administrator, and delegated users receive events for mailboxes they can access.
diff --git a/drizzle/migrations/0030_add_message_search_index.sql b/drizzle/migrations/0030_add_message_search_index.sql
new file mode 100644
index 0000000..904bcf5
--- /dev/null
+++ b/drizzle/migrations/0030_add_message_search_index.sql
@@ -0,0 +1,29 @@
+-- Full-text search over messages. An external-content FTS5 index reads its
+-- text from the messages table, so bodies are not stored twice; triggers keep
+-- it in step with every insert, update and delete, whichever code path writes.
+CREATE VIRTUAL TABLE `messages_fts` USING fts5(
+ `subject`,
+ `from_addr`,
+ `to_addr`,
+ `cc_addr`,
+ `text_body`,
+ `html_body`,
+ content='messages',
+ content_rowid='rowid',
+ tokenize='unicode61 remove_diacritics 2'
+);--> statement-breakpoint
+CREATE TRIGGER `messages_fts_ai` AFTER INSERT ON `messages` BEGIN
+ INSERT INTO `messages_fts`(`rowid`, `subject`, `from_addr`, `to_addr`, `cc_addr`, `text_body`, `html_body`)
+ VALUES (new.`rowid`, new.`subject`, new.`from_addr`, new.`to_addr`, new.`cc_addr`, new.`text_body`, new.`html_body`);
+END;--> statement-breakpoint
+CREATE TRIGGER `messages_fts_ad` AFTER DELETE ON `messages` BEGIN
+ INSERT INTO `messages_fts`(`messages_fts`, `rowid`, `subject`, `from_addr`, `to_addr`, `cc_addr`, `text_body`, `html_body`)
+ VALUES ('delete', old.`rowid`, old.`subject`, old.`from_addr`, old.`to_addr`, old.`cc_addr`, old.`text_body`, old.`html_body`);
+END;--> statement-breakpoint
+CREATE TRIGGER `messages_fts_au` AFTER UPDATE OF `subject`, `from_addr`, `to_addr`, `cc_addr`, `text_body`, `html_body` ON `messages` BEGIN
+ INSERT INTO `messages_fts`(`messages_fts`, `rowid`, `subject`, `from_addr`, `to_addr`, `cc_addr`, `text_body`, `html_body`)
+ VALUES ('delete', old.`rowid`, old.`subject`, old.`from_addr`, old.`to_addr`, old.`cc_addr`, old.`text_body`, old.`html_body`);
+ INSERT INTO `messages_fts`(`rowid`, `subject`, `from_addr`, `to_addr`, `cc_addr`, `text_body`, `html_body`)
+ VALUES (new.`rowid`, new.`subject`, new.`from_addr`, new.`to_addr`, new.`cc_addr`, new.`text_body`, new.`html_body`);
+END;--> statement-breakpoint
+INSERT INTO `messages_fts`(`messages_fts`) VALUES ('rebuild');
diff --git a/drizzle/migrations/meta/_journal.json b/drizzle/migrations/meta/_journal.json
index 0f60979..514ad9a 100644
--- a/drizzle/migrations/meta/_journal.json
+++ b/drizzle/migrations/meta/_journal.json
@@ -204,6 +204,20 @@
"when": 1788850800000,
"tag": "0028_add_keyboard_shortcuts_setting",
"breakpoints": true
+ },
+ {
+ "idx": 29,
+ "version": "6",
+ "when": 1788850800001,
+ "tag": "0029_add_spam_protection",
+ "breakpoints": true
+ },
+ {
+ "idx": 30,
+ "version": "6",
+ "when": 1788850800002,
+ "tag": "0030_add_message_search_index",
+ "breakpoints": true
}
]
}
diff --git a/src/app/api/admin/search-index/route.ts b/src/app/api/admin/search-index/route.ts
new file mode 100644
index 0000000..c718167
--- /dev/null
+++ b/src/app/api/admin/search-index/route.ts
@@ -0,0 +1,32 @@
+import { NextResponse } from "next/server";
+import { assertAdmin } from "@/lib/auth/admin";
+import { requireSessionUser } from "@/lib/api/auth";
+import { getEnv } from "@/lib/cloudflare";
+import { getSearchIndexStatus, rebuildSearchIndex } from "@/lib/search/index-admin";
+
+/** Row counts for the index versus the messages table, to spot drift. */
+export async function GET(request: Request) {
+ const env = getEnv();
+ const auth = await requireSessionUser(env, request);
+ if (auth.error) return auth.error;
+ try {
+ assertAdmin(auth.user);
+ } catch {
+ return NextResponse.json({ error: "Forbidden" }, { status: 403 });
+ }
+ return NextResponse.json(await getSearchIndexStatus(env));
+}
+
+/** Re-index every message from the messages table. Safe to run at any time. */
+export async function POST(request: Request) {
+ const env = getEnv();
+ const auth = await requireSessionUser(env, request);
+ if (auth.error) return auth.error;
+ try {
+ assertAdmin(auth.user);
+ } catch {
+ return NextResponse.json({ error: "Forbidden" }, { status: 403 });
+ }
+ await rebuildSearchIndex(env);
+ return NextResponse.json(await getSearchIndexStatus(env));
+}
diff --git a/src/app/api/messages/route.ts b/src/app/api/messages/route.ts
index 7025385..963b995 100644
--- a/src/app/api/messages/route.ts
+++ b/src/app/api/messages/route.ts
@@ -1,5 +1,5 @@
import { NextResponse } from "next/server";
-import { eq, desc, and, like, or, count, countDistinct, isNull, isNotNull, inArray, lte, gt, max, notInArray, sql, sum } from "drizzle-orm";
+import { eq, desc, and, or, count, countDistinct, isNull, isNotNull, inArray, lte, gt, max, notInArray, sql, sum } from "drizzle-orm";
import type { SQL } from "drizzle-orm";
import { getEnv } from "@/lib/cloudflare";
import { getCurrentUser } from "@/lib/auth/cookies";
@@ -9,6 +9,7 @@ import { getContactDisplayNameMap } from "@/lib/contacts/service";
import { getFirstEmailAddressEntry, normalizeEmailAddress } from "@/lib/email/address";
import { buildSnippet } from "@/lib/email/parse";
import { getMailboxAccessLevel, listAccessibleMailboxes } from "@/lib/mailboxes/access";
+import { buildSearchConditions } from "@/lib/search/conditions";
export async function GET(request: Request) {
const env = getEnv();
@@ -75,18 +76,10 @@ export async function GET(request: Request) {
if (read === "unread") {
conditions.push(eq(messages.read, false));
}
- if (query) {
- const pattern = `%${query}%`;
- const queryCondition = or(
- like(messages.fromAddr, pattern),
- like(messages.toAddr, pattern),
- like(messages.subject, pattern),
- like(messages.snippet, pattern),
- );
- if (queryCondition) conditions.push(queryCondition);
- }
- if (title) {
- conditions.push(like(messages.subject, `%${title}%`));
+ if (query || title) {
+ // Operators (from:, has:attachment, before:) and free text go through the
+ // full-text index; `title` is the legacy subject filter and is folded in.
+ conditions.push(...buildSearchConditions(title ? `${query ?? ""} subject:"${title}"` : query ?? ""));
}
const where = and(...conditions);
// Messages that were never threaded (older rows, drafts) stand alone.
diff --git a/src/app/api/v1/messages/route.ts b/src/app/api/v1/messages/route.ts
index 17b8504..08b8a13 100644
--- a/src/app/api/v1/messages/route.ts
+++ b/src/app/api/v1/messages/route.ts
@@ -6,6 +6,7 @@ import { authenticateApiKey, requireScope } from "@/lib/api/auth";
import { getDb } from "@/db";
import { messages, users } from "@/db/schema";
import { getMailboxAccessLevel, listAccessibleMailboxIds } from "@/lib/mailboxes/access";
+import { buildSearchConditions } from "@/lib/search/conditions";
export async function GET(request: Request) {
const env = getEnv();
@@ -17,6 +18,7 @@ export async function GET(request: Request) {
const url = new URL(request.url);
const mailboxId = url.searchParams.get("mailboxId");
const direction = url.searchParams.get("direction");
+ const query = url.searchParams.get("q")?.trim();
const limit = Math.min(Number(url.searchParams.get("limit") ?? 50), 100);
const db = getDb(env);
@@ -42,6 +44,7 @@ export async function GET(request: Request) {
if (direction === "inbound" || direction === "outbound") {
conditions.push(eq(messages.direction, direction));
}
+ if (query) conditions.push(...buildSearchConditions(query));
const rows = await db
.select()
diff --git a/src/components/mail-search/mail-search-context.tsx b/src/components/mail-search/mail-search-context.tsx
index 41672fd..ee9fbd2 100644
--- a/src/components/mail-search/mail-search-context.tsx
+++ b/src/components/mail-search/mail-search-context.tsx
@@ -1,6 +1,6 @@
"use client";
-import { createContext, useContext, useState } from "react";
+import { createContext, useContext, useEffect, useState } from "react";
import type { ReactNode } from "react";
import type { MailSearchContextValue } from "./types";
@@ -12,11 +12,23 @@ export function useMailSearch() {
return ctx;
}
+const SEARCH_DEBOUNCE_MS = 250;
+
export function MailSearchProvider({ children }: { children: ReactNode }) {
+ // `input` follows every keystroke; `query` is what lists fetch with, and
+ // trails it slightly so a typed word costs one request instead of one per letter.
+ const [input, setInput] = useState("");
const [query, setQuery] = useState("");
+ useEffect(() => {
+ // Clearing the box should empty the list at once; typing waits for a pause.
+ const delay = input.trim() ? SEARCH_DEBOUNCE_MS : 0;
+ const timer = setTimeout(() => setQuery(input.trim() ? input : ""), delay);
+ return () => clearTimeout(timer);
+ }, [input]);
+
return (
- <MailSearchContext.Provider value={{ query, setQuery }}>
+ <MailSearchContext.Provider value={{ query, input, setQuery: setInput }}>
{children}
</MailSearchContext.Provider>
);
diff --git a/src/components/mail-search/mail-search-input.tsx b/src/components/mail-search/mail-search-input.tsx
index ef22f77..39d5be9 100644
--- a/src/components/mail-search/mail-search-input.tsx
+++ b/src/components/mail-search/mail-search-input.tsx
@@ -6,7 +6,7 @@ import { useMailSearch } from "./mail-search-context";
import { useShortcuts } from "@/components/shortcuts";
export function MailSearchInput() {
- const { query, setQuery } = useMailSearch();
+ const { input: query, setQuery } = useMailSearch();
const { openCommandPalette, shortcutsEnabled, shortcutsPreferenceLoading } = useShortcuts();
const showShortcutHints = shortcutsEnabled && !shortcutsPreferenceLoading;
diff --git a/src/components/mail-search/types.d.ts b/src/components/mail-search/types.d.ts
index 73de578..1f34208 100644
--- a/src/components/mail-search/types.d.ts
+++ b/src/components/mail-search/types.d.ts
@@ -1,4 +1,7 @@
export type MailSearchContextValue = {
+ /** Debounced value lists fetch with. */
query: string;
+ /** Live value of the search box. */
+ input: string;
setQuery: (query: string) => void;
};
diff --git a/src/hooks/utils.ts b/src/hooks/utils.ts
index c87be29..f54e82e 100644
--- a/src/hooks/utils.ts
+++ b/src/hooks/utils.ts
@@ -1,30 +1,21 @@
import { authFetch } from "@/lib/auth/client";
+import { parseSearchQuery } from "@/lib/search/query-utils";
import type { MessageFilterOptions, MessageFolder } from "./types";
import type { MessageCounts, MessageListResponse } from "./types";
export const MESSAGE_POLL_INTERVAL_MS = 15_000;
+/**
+ * The search string goes to the server whole; operators are parsed there
+ * against the full-text index. Only the read state is lifted out here so the
+ * unread toggle and `is:unread` share one `read` parameter.
+ */
export function parseMessageSearchQuery(query: string): MessageFilterOptions {
- let remaining = query;
+ const parsed = parseSearchQuery(query);
const filters: MessageFilterOptions = {};
- const titleMatch = remaining.match(/\btitle:"([^"]+)"/i) ?? remaining.match(/\btitle:([^\s]+)/i);
-
- if (titleMatch?.[1]) {
- filters.title = titleMatch[1].trim();
- remaining = remaining.replace(titleMatch[0], " ");
- }
-
- if (/(^|\s):unread(\s|$)/i.test(remaining)) {
- filters.read = "unread";
- remaining = remaining.replace(/(^|\s):unread(?=\s|$)/gi, " ");
- } else if (/(^|\s):read(\s|$)/i.test(remaining)) {
- filters.read = "read";
- remaining = remaining.replace(/(^|\s):read(?=\s|$)/gi, " ");
- }
-
- const textQuery = remaining.replace(/\s+/g, " ").trim();
- if (textQuery) filters.query = textQuery;
-
+ if (parsed.read) filters.read = parsed.read;
+ const remaining = query.replace(/(^|\s)(is:(un)?read|:(un)?read)(?=\s|$)/gi, " ").replace(/\s+/g, " ").trim();
+ if (remaining) filters.query = remaining;
return filters;
}
diff --git a/src/lib/backups/export.ts b/src/lib/backups/export.ts
index e97ebff..02cc3d2 100644
--- a/src/lib/backups/export.ts
+++ b/src/lib/backups/export.ts
@@ -1,7 +1,7 @@
import type { DatabaseBackupDocument, DatabaseBackupTable, DatabaseRecord } from "./types";
import { mergeLegacyMessageBodies } from "./utils";
-const BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings", "email_templates", "calendar_events", "auto_reply_deliveries", "spam_token_stats", "spam_reputation", "spam_feedback"];
+const BACKUP_TABLES: DatabaseBackupTable[] = ["users", "domains", "mailboxes", "mailbox_access", "contacts", "folders", "api_keys", "messages", "message_attachments", "outbound_jobs", "routing_rules", "webhooks", "webhook_deliveries", "sessions", "audit_logs", "backup_settings", "backups", "app_settings", "license_settings", "email_templates", "calendar_events", "auto_reply_deliveries", "spam_token_stats", "spam_reputation", "spam_feedback", "mailbox_aliases"];
/**
* Tables every backup document must contain. Tables added to BACKUP_TABLES
* after the format shipped are absent from older documents, so they stay
@@ -17,7 +17,11 @@ export function getBackupConfigurationStatus(_env?: CloudflareEnv) {
/**
* Tables D1 manages itself, which are intentionally absent from BACKUP_TABLES.
*/
-const INTERNAL_TABLE_PATTERNS = ["sqlite_%", "_cf%"];
+const INTERNAL_TABLE_PATTERNS = ["sqlite_%", "_cf%", "messages_fts%"];
+/**
+ * The search index is derived data: its triggers repopulate it as messages are
+ * restored, so it is neither exported nor part of the coverage check.
+ */
const INTERNAL_TABLES = ["d1_migrations"];
/**
diff --git a/src/lib/backups/types.d.ts b/src/lib/backups/types.d.ts
index ac6d5bf..2518b27 100644
--- a/src/lib/backups/types.d.ts
+++ b/src/lib/backups/types.d.ts
@@ -1,5 +1,5 @@
export type BackupScheduleType = "daily" | "weekly" | "monthly";
-export type DatabaseBackupTable = "users" | "domains" | "mailboxes" | "mailbox_access" | "contacts" | "folders" | "api_keys" | "messages" | "message_attachments" | "outbound_jobs" | "routing_rules" | "webhooks" | "webhook_deliveries" | "sessions" | "audit_logs" | "backup_settings" | "backups" | "app_settings" | "license_settings" | "email_templates" | "calendar_events" | "auto_reply_deliveries" | "spam_token_stats" | "spam_reputation" | "spam_feedback";
+export type DatabaseBackupTable = "users" | "domains" | "mailboxes" | "mailbox_access" | "contacts" | "folders" | "api_keys" | "messages" | "message_attachments" | "outbound_jobs" | "routing_rules" | "webhooks" | "webhook_deliveries" | "sessions" | "audit_logs" | "backup_settings" | "backups" | "app_settings" | "license_settings" | "email_templates" | "calendar_events" | "auto_reply_deliveries" | "spam_token_stats" | "spam_reputation" | "spam_feedback" | "mailbox_aliases";
export type DatabaseRecord = Record<string, string | number | null>;
export type DatabaseBackupDocument = { format: "mailflare-database-backup"; version: 1; createdAt: string; tables: Record<DatabaseBackupTable, DatabaseRecord[]>; };
diff --git a/src/lib/search/conditions.ts b/src/lib/search/conditions.ts
new file mode 100644
index 0000000..8eca20c
--- /dev/null
+++ b/src/lib/search/conditions.ts
@@ -0,0 +1,33 @@
+import { and, eq, exists, gte, lt, sql } from "drizzle-orm";
+import type { SQL } from "drizzle-orm";
+import { messageAttachments, messages } from "@/db/schema";
+import { buildFtsMatch, parseSearchQuery } from "./query-utils";
+
+/**
+ * Translate a search string into WHERE conditions on `messages`. Text goes to
+ * the FTS5 index by rowid; everything else is a plain column predicate, so the
+ * caller ANDs these with its own scope (mailbox access, folder, status).
+ */
+export function buildSearchConditions(raw: string): SQL[] {
+ const parsed = parseSearchQuery(raw);
+ const conditions: SQL[] = [];
+
+ const match = buildFtsMatch(parsed);
+ if (match) {
+ conditions.push(sql`${messages}.rowid IN (SELECT rowid FROM messages_fts WHERE messages_fts MATCH ${match})`);
+ }
+ if (parsed.hasAttachment) {
+ conditions.push(
+ exists(
+ sql`(SELECT 1 FROM ${messageAttachments} WHERE ${messageAttachments.messageId} = ${messages.id} AND ${messageAttachments.disposition} = 'attachment')`,
+ ),
+ );
+ }
+ if (parsed.read === "read") conditions.push(eq(messages.read, true));
+ if (parsed.read === "unread") conditions.push(eq(messages.read, false));
+ if (parsed.starred) conditions.push(eq(messages.starred, true));
+ if (parsed.after) conditions.push(gte(messages.createdAt, parsed.after));
+ if (parsed.before) conditions.push(lt(messages.createdAt, parsed.before));
+
+ return conditions.length ? [and(...conditions)!] : [];
+}
diff --git a/src/lib/search/index-admin.ts b/src/lib/search/index-admin.ts
new file mode 100644
index 0000000..3025c59
--- /dev/null
+++ b/src/lib/search/index-admin.ts
@@ -0,0 +1,16 @@
+/**
+ * Maintenance for the FTS5 index. Triggers keep it current, so a rebuild is
+ * only needed after a restore from a backup that predates the index, or if the
+ * counts below ever disagree.
+ */
+export async function rebuildSearchIndex(env: CloudflareEnv): Promise<void> {
+ await env.DB.prepare("INSERT INTO messages_fts(messages_fts) VALUES ('rebuild')").run();
+}
+
+export async function getSearchIndexStatus(env: CloudflareEnv): Promise<{ indexed: number; messages: number }> {
+ const [indexed, total] = await env.DB.batch<{ n: number }>([
+ env.DB.prepare("SELECT count(*) AS n FROM messages_fts"),
+ env.DB.prepare("SELECT count(*) AS n FROM messages"),
+ ]);
+ return { indexed: indexed.results[0]?.n ?? 0, messages: total.results[0]?.n ?? 0 };
+}
diff --git a/src/lib/search/query-utils.ts b/src/lib/search/query-utils.ts
new file mode 100644
index 0000000..70de828
--- /dev/null
+++ b/src/lib/search/query-utils.ts
@@ -0,0 +1,140 @@
+import type { ParsedSearchQuery, SearchToken } from "./types";
+
+/**
+ * Gmail-style search grammar shared by the search box and the API:
+ *
+ * from:maya to:"sam okoro" subject:invoice has:attachment is:unread
+ * is:starred after:2026-09-01 before:2026-09-30 "exact phrase" -excluded word
+ *
+ * The older `title:` and bare `:unread` / `:read` forms keep working.
+ */
+const OPERATOR_RE = /(?:^|\s)(from|to|subject|title|has|is|after|before|newer|older):(?:"([^"]*)"|(\S+))/gi;
+
+export function parseSearchQuery(raw: string): ParsedSearchQuery {
+ const parsed: ParsedSearchQuery = { text: "", needsFullText: false };
+ let rest = raw ?? "";
+
+ rest = rest.replace(OPERATOR_RE, (_match, key: string, quoted: string | undefined, bare: string | undefined) => {
+ const value = (quoted ?? bare ?? "").trim();
+ switch (key.toLowerCase()) {
+ case "from":
+ parsed.from = value;
+ break;
+ case "to":
+ parsed.to = value;
+ break;
+ case "subject":
+ case "title":
+ parsed.subject = value;
+ break;
+ case "has":
+ if (/^attachments?$/i.test(value)) parsed.hasAttachment = true;
+ break;
+ case "is":
+ if (/^unread$/i.test(value)) parsed.read = "unread";
+ else if (/^read$/i.test(value)) parsed.read = "read";
+ else if (/^starred$/i.test(value)) parsed.starred = true;
+ break;
+ case "after":
+ case "newer":
+ parsed.after = parseDate(value) ?? parsed.after;
+ break;
+ case "before":
+ case "older":
+ parsed.before = parseDate(value) ?? parsed.before;
+ break;
+ }
+ return " ";
+ });
+
+ if (/(^|\s):unread(?=\s|$)/i.test(rest)) {
+ parsed.read = "unread";
+ rest = rest.replace(/(^|\s):unread(?=\s|$)/gi, " ");
+ } else if (/(^|\s):read(?=\s|$)/i.test(rest)) {
+ parsed.read = "read";
+ rest = rest.replace(/(^|\s):read(?=\s|$)/gi, " ");
+ }
+
+ parsed.text = rest.replace(/\s+/g, " ").trim();
+ parsed.needsFullText = !!(parsed.text || parsed.from || parsed.to || parsed.subject);
+ return parsed;
+}
+
+function parseDate(value: string): Date | undefined {
+ const match = value.match(/^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$/);
+ if (!match) return undefined;
+ const date = new Date(Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3])));
+ return Number.isNaN(date.getTime()) ? undefined : date;
+}
+
+/** Split free text into phrases ("..."), negations (-word) and plain terms. */
+export function tokenizeSearchText(text: string): SearchToken[] {
+ const tokens: SearchToken[] = [];
+ const re = /(-?)"([^"]+)"|(-?)(\S+)/g;
+ let match: RegExpExecArray | null;
+ while ((match = re.exec(text))) {
+ if (match[2] !== undefined) {
+ const term = match[2].trim();
+ if (term) tokens.push({ term, negate: match[1] === "-", phrase: true });
+ } else {
+ const term = match[4].replace(/^-+/, "").trim();
+ if (term) tokens.push({ term, negate: match[3] === "-", phrase: false });
+ }
+ }
+ return tokens;
+}
+
+/** FTS5 string literal: double quotes doubled, so user input can never alter the query grammar. */
+function ftsString(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`;
+}
+
+/**
+ * One term. Phrases match whole words; plain terms match by prefix so "inv"
+ * finds "invoice". Punctuation FTS5 treats as separators is normalised to
+ * spaces first, so "[email protected]" becomes the phrase "maya acme test".
+ */
+function ftsTerm(term: string, phrase: boolean): string {
+ const cleaned = term.replace(/[^\p{L}\p{N}\s]+/gu, " ").replace(/\s+/g, " ").trim();
+ if (!cleaned) return "";
+ if (phrase || cleaned.includes(" ")) {
+ // A multi-word prefix query needs the last word starred, which FTS5 only
+ // allows inside a phrase as `"a b" *`; keep it simple and require whole words.
+ return ftsString(cleaned);
+ }
+ return `${ftsString(cleaned)}*`;
+}
+
+/**
+ * Build the FTS5 MATCH expression. Column filters restrict a term to one
+ * column; free text is ANDed across the whole row. Returns null when there is
+ * nothing to match, so callers skip the index entirely.
+ */
+export function buildFtsMatch(parsed: ParsedSearchQuery): string | null {
+ const parts: string[] = [];
+
+ for (const token of tokenizeSearchText(parsed.text)) {
+ const expr = ftsTerm(token.term, token.phrase);
+ if (!expr) continue;
+ parts.push(token.negate ? `NOT ${expr}` : expr);
+ }
+ const column = (name: string, value: string | undefined) => {
+ if (!value) return;
+ const terms = tokenizeSearchText(value)
+ .map((token) => ftsTerm(token.term, token.phrase))
+ .filter(Boolean);
+ if (terms.length === 0) return;
+ parts.push(`${name} : (${terms.join(" AND ")})`);
+ };
+ column("subject", parsed.subject);
+ column("from_addr", parsed.from);
+ column("{to_addr cc_addr}", parsed.to);
+
+ if (parts.length === 0) return null;
+ // FTS5 has no unary NOT: "a NOT b" is binary. A leading negation is turned
+ // into "match anything" minus the term.
+ const positives = parts.filter((part) => !part.startsWith("NOT "));
+ const negatives = parts.filter((part) => part.startsWith("NOT ")).map((part) => part.slice(4));
+ const base = positives.length > 0 ? positives.join(" AND ") : `${ftsString("")}*`;
+ return negatives.length > 0 ? `(${base}) NOT (${negatives.join(" OR ")})` : base;
+}
diff --git a/src/lib/search/types.d.ts b/src/lib/search/types.d.ts
new file mode 100644
index 0000000..17b5818
--- /dev/null
+++ b/src/lib/search/types.d.ts
@@ -0,0 +1,18 @@
+export type ParsedSearchQuery = {
+ /** Free text after operators are removed; matched against every indexed column. */
+ text: string;
+ from?: string;
+ to?: string;
+ subject?: string;
+ hasAttachment?: boolean;
+ read?: "read" | "unread";
+ starred?: boolean;
+ /** Inclusive lower bound on the message date, from `after:YYYY-MM-DD`. */
+ after?: Date;
+ /** Exclusive upper bound on the message date, from `before:YYYY-MM-DD`. */
+ before?: Date;
+ /** True when anything in the query needs the full-text index. */
+ needsFullText: boolean;
+};
+
+export type SearchToken = { term: string; negate: boolean; phrase: boolean };
diff --git a/src/lib/setup/migration.ts b/src/lib/setup/migration.ts
index d91ec72..43d7d8f 100644
--- a/src/lib/setup/migration.ts
+++ b/src/lib/setup/migration.ts
@@ -29,6 +29,7 @@ const MIGRATION_NAMES = [
"0027_add_domain_sending_intent.sql",
"0028_add_keyboard_shortcuts_setting.sql",
"0029_add_spam_protection.sql",
+ "0030_add_message_search_index.sql",
];
const INITIAL_SCHEMA_SQL = `
@@ -61,6 +62,10 @@ CREATE TABLE IF NOT EXISTS messages (id text PRIMARY KEY NOT NULL, user_id text
CREATE INDEX IF NOT EXISTS messages_user_created_idx ON messages(user_id, created_at);
CREATE INDEX IF NOT EXISTS messages_mailbox_idx ON messages(mailbox_id);
CREATE INDEX IF NOT EXISTS messages_folder_idx ON messages(folder_id);
+CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(subject, from_addr, to_addr, cc_addr, text_body, html_body, content='messages', content_rowid='rowid', tokenize='unicode61 remove_diacritics 2');
+CREATE TRIGGER IF NOT EXISTS messages_fts_ai AFTER INSERT ON messages BEGIN INSERT INTO messages_fts(rowid, subject, from_addr, to_addr, cc_addr, text_body, html_body) VALUES (new.rowid, new.subject, new.from_addr, new.to_addr, new.cc_addr, new.text_body, new.html_body); END;
+CREATE TRIGGER IF NOT EXISTS messages_fts_ad AFTER DELETE ON messages BEGIN INSERT INTO messages_fts(messages_fts, rowid, subject, from_addr, to_addr, cc_addr, text_body, html_body) VALUES ('delete', old.rowid, old.subject, old.from_addr, old.to_addr, old.cc_addr, old.text_body, old.html_body); END;
+CREATE TRIGGER IF NOT EXISTS messages_fts_au AFTER UPDATE OF subject, from_addr, to_addr, cc_addr, text_body, html_body ON messages BEGIN INSERT INTO messages_fts(messages_fts, rowid, subject, from_addr, to_addr, cc_addr, text_body, html_body) VALUES ('delete', old.rowid, old.subject, old.from_addr, old.to_addr, old.cc_addr, old.text_body, old.html_body); INSERT INTO messages_fts(rowid, subject, from_addr, to_addr, cc_addr, text_body, html_body) VALUES (new.rowid, new.subject, new.from_addr, new.to_addr, new.cc_addr, new.text_body, new.html_body); END;
CREATE INDEX IF NOT EXISTS messages_thread_idx ON messages(mailbox_id, thread_id);
CREATE INDEX IF NOT EXISTS messages_provider_message_idx ON messages(mailbox_id, provider_message_id);
CREATE INDEX IF NOT EXISTS messages_raw_r2_key_idx ON messages(raw_r2_key);
@@ -116,14 +121,32 @@ export async function migrateCleanDatabase(db: D1Database): Promise<boolean> {
);
}
- const schemaStatements = INITIAL_SCHEMA_SQL
- .split(";")
- .map((statement) => statement.trim())
- .filter(Boolean)
- .map((statement) => db.prepare(statement));
+ const schemaStatements = splitSqlStatements(INITIAL_SCHEMA_SQL).map((statement) => db.prepare(statement));
const migrationStatements = MIGRATION_NAMES.map((name) =>
db.prepare("INSERT OR IGNORE INTO d1_migrations (name) VALUES (?)").bind(name),
);
await db.batch([...schemaStatements, ...migrationStatements]);
return true;
}
+
+/**
+ * Split the bootstrap SQL into statements. A plain split on ";" would cut a
+ * trigger body apart, so pieces are joined back together until every BEGIN
+ * has its END.
+ */
+export function splitSqlStatements(sqlText: string): string[] {
+ const statements: string[] = [];
+ let current = "";
+ for (const piece of sqlText.split(";")) {
+ current = current ? `${current};${piece}` : piece;
+ const opened = (current.match(/\bBEGIN\b/gi) ?? []).length;
+ const closed = (current.match(/\bEND\b/gi) ?? []).length;
+ if (opened > closed) continue;
+ const statement = current.trim();
+ if (statement) statements.push(statement);
+ current = "";
+ }
+ const rest = current.trim();
+ if (rest) statements.push(rest);
+ return statements;
+}