Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🔎 perf: Index-Bound Word-Prefix Search for Users and Groups (#16719)

Marco Berettade9d46fcd4f4
diff --git a/api/server/services/start/__tests__/migration.spec.js b/api/server/services/start/__tests__/migration.spec.js
new file mode 100644
index 0000000000..ba4bfc32d8
--- /dev/null
+++ b/api/server/services/start/__tests__/migration.spec.js
@@ -0,0 +1,37 @@
+const mockWarnOnMissingSearchTokens = jest.fn();
+const mockCheckAgentPermissionsMigration = jest.fn();
+const mockCheckPromptPermissionsMigration = jest.fn();
+
+jest.mock('@librechat/data-schemas', () => ({
+ logger: { error: jest.fn() },
+ warnOnMissingSearchTokens: (...args) => mockWarnOnMissingSearchTokens(...args),
+}));
+jest.mock('@librechat/api', () => ({
+ logAgentMigrationWarning: jest.fn(),
+ logPromptMigrationWarning: jest.fn(),
+ checkAgentPermissionsMigration: (...args) => mockCheckAgentPermissionsMigration(...args),
+ checkPromptPermissionsMigration: (...args) => mockCheckPromptPermissionsMigration(...args),
+}));
+jest.mock('~/models', () => ({ findRoleByIdentifier: jest.fn() }));
+
+const { checkMigrations } = require('../migration');
+
+describe('checkMigrations', () => {
+ it('starts the search token probe while the permission checks are still running', async () => {
+ let finishAgentCheck;
+ mockCheckAgentPermissionsMigration.mockReturnValue(
+ new Promise((resolve) => {
+ finishAgentCheck = resolve;
+ }),
+ );
+ mockCheckPromptPermissionsMigration.mockResolvedValue({});
+ mockWarnOnMissingSearchTokens.mockResolvedValue(undefined);
+
+ const done = checkMigrations();
+ expect(mockWarnOnMissingSearchTokens).toHaveBeenCalledTimes(1);
+
+ finishAgentCheck({});
+ await done;
+ expect(mockCheckPromptPermissionsMigration).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/api/server/services/start/migration.js b/api/server/services/start/migration.js
index 70f8300e08..d170590168 100644
--- a/api/server/services/start/migration.js
+++ b/api/server/services/start/migration.js
@@ -1,5 +1,5 @@
const mongoose = require('mongoose');
-const { logger } = require('@librechat/data-schemas');
+const { logger, warnOnMissingSearchTokens } = require('@librechat/data-schemas');
const {
logAgentMigrationWarning,
logPromptMigrationWarning,
@@ -13,6 +13,8 @@ const { findRoleByIdentifier } = require('~/models');
* This runs at the end to ensure all systems are initialized
*/
async function checkMigrations() {
+ /** Started first so it overlaps the permission checks; it logs its own failures. */
+ const searchTokens = warnOnMissingSearchTokens(mongoose.connection);
try {
const agentMigrationResult = await checkAgentPermissionsMigration({
mongoose,
@@ -37,6 +39,7 @@ async function checkMigrations() {
} catch (error) {
logger.error('Failed to check prompt permissions migration:', error);
}
+ await searchTokens;
}
module.exports = {
diff --git a/config/migrate-search-tokens.js b/config/migrate-search-tokens.js
new file mode 100644
index 0000000000..812e0e0338
--- /dev/null
+++ b/config/migrate-search-tokens.js
@@ -0,0 +1,30 @@
+require('dotenv').config();
+const mongoose = require('mongoose');
+const { backfillSearchTokens } = require('@librechat/data-schemas');
+const connect = require('./connect');
+
+/**
+ * Backfills the word-prefix search tokens on users and groups saved before they existed.
+ * Idempotent; safe to run while the server is up.
+ *
+ * Usage: npm run migrate:search-tokens [-- --dry-run] [-- --batch-size=500]
+ */
+(async () => {
+ let exitCode = 0;
+ try {
+ await connect();
+ const batchArg = process.argv.find((arg) => arg.startsWith('--batch-size='));
+ const result = await backfillSearchTokens(mongoose.connection, {
+ dryRun: process.argv.includes('--dry-run'),
+ batchSize: batchArg ? parseInt(batchArg.split('=')[1], 10) || undefined : undefined,
+ });
+ console.log(JSON.stringify(result, null, 2));
+ } catch (error) {
+ console.error('Search token migration failed:', error);
+ exitCode = 1;
+ } finally {
+ await mongoose.disconnect();
+ }
+ /** Exit explicitly: with Redis enabled, the cache clients opened on connect keep the process alive. */
+ process.exit(exitCode);
+})();
diff --git a/e2e/specs/mock/db.ts b/e2e/specs/mock/db.ts
index 40127f7135..4a0492eb7a 100644
--- a/e2e/specs/mock/db.ts
+++ b/e2e/specs/mock/db.ts
@@ -16,7 +16,7 @@ function getRuntimeEnvPath(): string {
);
}
-function getMongoUri(): string {
+export function getMongoUri(): string {
try {
const env = JSON.parse(fs.readFileSync(getRuntimeEnvPath(), 'utf8')) as { MONGO_URI?: string };
if (env.MONGO_URI) {
diff --git a/e2e/specs/mock/scenarios/people-search-tokens.spec.ts b/e2e/specs/mock/scenarios/people-search-tokens.spec.ts
new file mode 100644
index 0000000000..5bef523e2b
--- /dev/null
+++ b/e2e/specs/mock/scenarios/people-search-tokens.spec.ts
@@ -0,0 +1,226 @@
+import path from 'path';
+import { promisify } from 'util';
+import { execFile } from 'child_process';
+import { expect, test } from '@playwright/test';
+import type { Page, Locator } from '@playwright/test';
+import { MOCK_ENDPOINTS, NEW_CHAT_PATH, getAccessToken, requestJson, uniqueName } from '../helpers';
+import { openAgentBuilder, uniqueAgentName, cleanupAgent } from '../agents.helpers';
+import { withMongo, getMongoUri } from '../db';
+
+/**
+ * People search (`GET /api/permissions/search-principals`, the share dialog's
+ * picker) matches word prefixes over the derived `nameTokens` / `emailTokens` /
+ * `usernameTokens` arrays. These scenarios register users through the real
+ * auth route, so the schema middleware writes the tokens, and drive the
+ * backfill script (`npm run migrate:search-tokens`) against this run's database
+ * for documents a writer left without tokens or with stale ones.
+ *
+ * The primary e2e user is the first one registered in the run, so it is ADMIN
+ * and may search principals and share agents without seeding role permissions
+ * (see `principal-type-labels.spec.ts`).
+ */
+
+const SEARCH_LABEL = 'Search for people or groups by name or email';
+const SEARCH_PATH = '/api/permissions/search-principals';
+const REPO_ROOT = path.resolve(__dirname, '../../../..');
+
+type SearchResult = { name?: string; email?: string; type: string };
+
+/** Letters only, so a suffix stays one search word. */
+function suffix(): string {
+ return Array.from({ length: 7 }, () =>
+ String.fromCharCode(97 + Math.floor(Math.random() * 26)),
+ ).join('');
+}
+
+async function register(page: Page, name: string, email: string, username: string) {
+ const password = 'Search-tokens-pass-1';
+ const response = await page.request.post('/api/auth/register', {
+ data: { name, email, username, password, confirm_password: password },
+ });
+ expect(response.ok(), await response.text()).toBeTruthy();
+}
+
+async function search(page: Page, token: string, q: string): Promise<SearchResult[]> {
+ const body = await requestJson<{ results: SearchResult[] }>(page, {
+ path: `${SEARCH_PATH}?q=${encodeURIComponent(q)}&limit=20`,
+ token,
+ });
+ return body.results;
+}
+
+async function deleteUsers(emails: string[]): Promise<void> {
+ await withMongo(async (db) => {
+ await db.collection('users').deleteMany({ email: { $in: emails } });
+ });
+}
+
+async function openShareDialog(page: Page): Promise<{ dialog: Locator; agentId: string }> {
+ const agentName = uniqueAgentName('E2E People Search');
+ const token = await getAccessToken(page);
+ const agent = await requestJson<{ id: string }>(page, {
+ path: '/api/agents',
+ token,
+ method: 'POST',
+ body: { name: agentName, provider: MOCK_ENDPOINTS[0].label, model: MOCK_ENDPOINTS[0].model },
+ });
+ const form = await openAgentBuilder(page);
+ await form.getByRole('combobox', { name: 'Agent', exact: true }).click();
+ await page.getByRole('option', { name: agentName, exact: true }).click();
+ await expect(form.getByLabel('Agent name')).toHaveValue(agentName);
+ await page.getByRole('button', { name: `Share ${agentName}` }).click();
+ const dialog = page.getByRole('dialog', { name: `Share ${agentName}` });
+ await expect(dialog).toBeVisible();
+ return { dialog, agentId: agent.id };
+}
+
+const execFileAsync = promisify(execFile);
+
+/** Async so a stuck script fails the test on its timeout instead of blocking the worker. */
+async function runBackfill(): Promise<void> {
+ try {
+ await execFileAsync('node', ['config/migrate-search-tokens.js'], {
+ cwd: REPO_ROOT,
+ env: { ...process.env, MONGO_URI: getMongoUri() },
+ timeout: 60000,
+ killSignal: 'SIGKILL',
+ });
+ } catch (error) {
+ const { stdout = '', stderr = '' } = error as { stdout?: string; stderr?: string };
+ throw new Error(
+ `Backfill failed: ${String(error)}\n${stdout.slice(-2000)}\n${stderr.slice(-2000)}`,
+ );
+ }
+}
+
+test.describe('people search over word-prefix tokens', () => {
+ test('@scenario:people-search-matches-word-prefixes the share dialog finds a user by the start of each name word, and not by a fragment inside a word', async ({
+ page,
+ }) => {
+ test.setTimeout(120000);
+ const id = suffix();
+ const user = { name: `Quill${id} Brightwater`, email: `${uniqueName('quill')}@search.test` };
+ let agentId: string | undefined;
+ try {
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await register(page, user.name, user.email, `quill${id}`);
+
+ const opened = await openShareDialog(page);
+ agentId = opened.agentId;
+ const searchBox = opened.dialog.getByRole('combobox', { name: SEARCH_LABEL });
+ await searchBox.fill(`brig quill${id.slice(0, 3)}`);
+ await expect(opened.dialog.getByRole('option').filter({ hasText: user.name })).toBeVisible({
+ timeout: 10000,
+ });
+
+ const token = await getAccessToken(page);
+ expect((await search(page, token, `ill${id}`)).map((r) => r.email)).not.toContain(user.email);
+ expect((await search(page, token, user.email.slice(0, 12))).map((r) => r.email)).toContain(
+ user.email,
+ );
+ } finally {
+ await cleanupAgent(page, agentId);
+ await deleteUsers([user.email]);
+ }
+ });
+
+ test('@scenario:people-search-folds-accents-and-ranks-exact-first an unaccented query finds an accented name and ranks its exact match above a longer prefix match', async ({
+ page,
+ }) => {
+ const id = suffix();
+ const exact = { name: `Zoë${id}`, email: `${uniqueName('zoe-exact')}@search.test` };
+ const prefix = { name: `Zoe${id}x Park`, email: `${uniqueName('zoe-prefix')}@search.test` };
+ try {
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await register(page, prefix.name, prefix.email, `zoep${id}`);
+ await register(page, exact.name, exact.email, `zoee${id}`);
+
+ const token = await getAccessToken(page);
+ const results = await search(page, token, `zoe${id}`);
+ expect(results.map((r) => r.name)).toEqual([exact.name, prefix.name]);
+ } finally {
+ await deleteUsers([exact.email, prefix.email]);
+ }
+ });
+
+ test('@scenario:backfill-repairs-missing-and-stale-tokens a user saved without tokens is still found, a user whose tokens went stale is found under the new name once the backfill runs', async ({
+ page,
+ }) => {
+ test.setTimeout(120000);
+ const id = suffix();
+ const legacy = { name: `Legacy${id} Person`, email: `${uniqueName('legacy')}@search.test` };
+ const renamed = { name: `Renamed${id} Person`, email: `${uniqueName('renamed')}@search.test` };
+ try {
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ const primary = await withMongo((db) =>
+ db.collection('users').findOne({}, { sort: { createdAt: 1 } }),
+ );
+ /** Raw writes, as an older server would leave them: one user with no
+ * tokens at all, one renamed without its tokens being rewritten. */
+ await withMongo(async (db) => {
+ const now = new Date();
+ await db.collection('users').insertMany([
+ { ...legacy, tenantId: primary?.tenantId, createdAt: now, updatedAt: now },
+ {
+ ...renamed,
+ tenantId: primary?.tenantId,
+ nameTokens: [`original${id}`, 'person'],
+ emailTokens: [renamed.email],
+ usernameTokens: [],
+ createdAt: now,
+ updatedAt: now,
+ },
+ ]);
+ });
+
+ const token = await getAccessToken(page);
+ const emails = async (q: string) => (await search(page, token, q)).map((r) => r.email);
+ expect(await emails(`gacy${id}`)).toContain(legacy.email);
+ expect(await emails(`renamed${id}`)).not.toContain(renamed.email);
+
+ await runBackfill();
+
+ expect(await emails(`renamed${id}`)).toContain(renamed.email);
+ expect(await emails(`original${id}`)).not.toContain(renamed.email);
+ expect(await emails(`legacy${id}`)).toContain(legacy.email);
+ const stored = await withMongo((db) =>
+ db.collection('users').findOne({ email: legacy.email }),
+ );
+ expect(stored?.nameTokens).toEqual([`legacy${id}`, 'person']);
+ } finally {
+ await deleteUsers([legacy.email, renamed.email]);
+ }
+ });
+
+ test('@scenario:people-picker-searches-once-per-pause typing a query character by character sends one search request, not one per keystroke', async ({
+ page,
+ }) => {
+ test.setTimeout(120000);
+ const id = suffix();
+ const user = { name: `Pause${id} Typist`, email: `${uniqueName('pause')}@search.test` };
+ let agentId: string | undefined;
+ try {
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await register(page, user.name, user.email, `pause${id}`);
+ const opened = await openShareDialog(page);
+ agentId = opened.agentId;
+
+ const queries: string[] = [];
+ page.on('request', (request) => {
+ const url = new URL(request.url());
+ if (url.pathname === SEARCH_PATH) {
+ queries.push(url.searchParams.get('q') ?? '');
+ }
+ });
+ const searchBox = opened.dialog.getByRole('combobox', { name: SEARCH_LABEL });
+ await searchBox.pressSequentially(`pause${id}`, { delay: 40 });
+ await expect(opened.dialog.getByRole('option').filter({ hasText: user.name })).toBeVisible({
+ timeout: 10000,
+ });
+ expect(queries).toEqual([`pause${id}`]);
+ } finally {
+ await cleanupAgent(page, agentId);
+ await deleteUsers([user.email]);
+ }
+ });
+});
diff --git a/package.json b/package.json
index 468915b789..72dde5ff7f 100644
--- a/package.json
+++ b/package.json
@@ -143,6 +143,8 @@
"migrate:tenant-indexes": "node config/migrate-tenant-indexes.js",
"migrate:tenant-indexes:dry-run": "node config/migrate-tenant-indexes.js --dry-run",
"migrate:terms-timestamp": "node config/migrate-terms-timestamp.js",
+ "migrate:search-tokens": "node config/migrate-search-tokens.js",
+ "migrate:search-tokens:dry-run": "node config/migrate-search-tokens.js --dry-run",
"flush-cache": "node config/flush-cache.js",
"migrate:agent-permissions:dry-run": "node config/migrate-agent-permissions.js --dry-run",
"migrate:agent-permissions": "node config/migrate-agent-permissions.js",
diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.ts
index 3c1aea1303..b3582c33f1 100644
--- a/packages/api/src/admin/users.spec.ts
+++ b/packages/api/src/admin/users.spec.ts
@@ -1,4 +1,5 @@
import { Types } from 'mongoose';
+import { buildUserSearchFilter } from '@librechat/data-schemas';
import { PrincipalType, SystemRoles } from 'librechat-data-provider';
import type { IUser, UserDeleteResult } from '@librechat/data-schemas';
import type { Response } from 'express';
@@ -205,20 +206,23 @@ describe('createAdminUsersHandlers', () => {
expect(response.capped).toBe(true);
});
- it('searches name, email, and username with anchored prefix regex', async () => {
+ it('searches the indexed word-prefix tokens of name, email, and username', async () => {
const findUsers = jest.fn().mockResolvedValue([]);
const deps = createDeps({ findUsers });
const handlers = createAdminUsersHandlers(deps);
- const { req, res } = createReqRes({ query: { q: 'test' } });
+ const { req, res } = createReqRes({ query: { q: 'Test' } });
await handlers.searchUsers(req, res);
const filter = findUsers.mock.calls[0][0];
- expect(filter.$or).toHaveLength(3);
- expect(filter.$or[0]).toHaveProperty('name');
- expect(filter.$or[1]).toHaveProperty('email');
- expect(filter.$or[2]).toHaveProperty('username');
- expect(filter.$or[0].name.source).toBe('^test');
+ expect(filter).toEqual(buildUserSearchFilter('Test', { legacyPrefix: true }));
+ expect(filter.$or).toEqual(
+ expect.arrayContaining([
+ { nameTokens: /^test/ },
+ { emailTokens: /^test/ },
+ { usernameTokens: /^test/ },
+ ]),
+ );
});
it('projects username in the field selection', async () => {
@@ -242,8 +246,25 @@ describe('createAdminUsersHandlers', () => {
await handlers.searchUsers(req, res);
const filter = findUsers.mock.calls[0][0];
- expect(filter.$or[0].name).toBeInstanceOf(RegExp);
- expect(filter.$or[0].name.source).toBe('^test\\.user\\+1');
+ expect(filter.$and[0].$or).toContainEqual({ emailTokens: /^test\.user\+1/ });
+ /** Documents awaiting the backfill keep the old anchored prefix match. */
+ expect(filter.$and[0].$or).toContainEqual({
+ nameTokens: { $exists: false },
+ name: /^test\.user\+1/i,
+ });
+ });
+
+ it('returns no users without querying when the query has nothing to match', async () => {
+ const findUsers = jest.fn();
+ const deps = createDeps({ findUsers });
+ const handlers = createAdminUsersHandlers(deps);
+ const { req, res, status, json } = createReqRes({ query: { q: '\u0301\u0301' } });
+
+ await handlers.searchUsers(req, res);
+
+ expect(findUsers).not.toHaveBeenCalled();
+ expect(status).toHaveBeenCalledWith(200);
+ expect(json).toHaveBeenCalledWith({ users: [], total: 0, capped: false });
});
it('returns 400 when query is missing', async () => {
diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.ts
index ae8ac19c9c..d28569e50a 100644
--- a/packages/api/src/admin/users.ts
+++ b/packages/api/src/admin/users.ts
@@ -1,7 +1,7 @@
import { Types } from 'mongoose';
import { randomUUID } from 'node:crypto';
import { PrincipalType, SystemRoles } from 'librechat-data-provider';
-import { logger, isValidObjectIdString } from '@librechat/data-schemas';
+import { logger, isValidObjectIdString, buildUserSearchFilter } from '@librechat/data-schemas';
import type {
IUser,
IConfig,
@@ -197,14 +197,13 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
}
const searchLimit = Math.min(Math.max(1, parseInt(limitStr, 10) || 20), 50);
- const escaped = trimmed.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
- const regex = new RegExp(`^${escaped}`, 'i');
-
- const users = await findUsers(
- { $or: [{ name: regex }, { email: regex }, { username: regex }] },
- '_id name email username avatar',
- { limit: searchLimit, sort: { name: 1 } },
- );
+ const filter = buildUserSearchFilter(trimmed, { legacyPrefix: true });
+ const users = filter
+ ? await findUsers(filter, '_id name email username avatar', {
+ limit: searchLimit,
+ sort: { name: 1 },
+ })
+ : [];
const results: AdminUserSearchResult[] = users.map((u) => ({
id: u._id?.toString() ?? '',
diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.ts
index 560d21ebdb..38b269be70 100644
--- a/packages/data-schemas/src/index.ts
+++ b/packages/data-schemas/src/index.ts
@@ -117,4 +117,7 @@ export {
migrateTenantIndexes,
dropSupersededPromptGroupIndexes,
backfillMCPServerNormalizedNames,
+ backfillSearchTokens,
+ warnOnMissingSearchTokens,
} from './migrations';
+export type { SearchTokenBackfillResult } from './migrations';
diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts
index d9318173d1..7bffe5e381 100644
--- a/packages/data-schemas/src/methods/user.methods.spec.ts
+++ b/packages/data-schemas/src/methods/user.methods.spec.ts
@@ -1842,6 +1842,35 @@ describe('User Methods - Database Tests', () => {
expect(results.length).toBeGreaterThan(0);
});
+ test('should rank an accent-folded exact match above a prefix match', async () => {
+ await User.create([
+ { name: 'Zoe Adams', email: '[email protected]', username: 'zadams', provider: 'local' },
+ { name: 'Zoë', email: '[email protected]', username: 'zumlaut', provider: 'local' },
+ ]);
+
+ const results = await methods.searchUsers({ searchPattern: 'zoe', limit: 1 });
+
+ expect(results).toHaveLength(1);
+ expect((results[0] as unknown as t.IUser).name).toBe('Zoë');
+ });
+
+ test('should rank a reordered full-name match above a longer prefix match', async () => {
+ await User.create([
+ {
+ name: 'Smith Johnson',
+ email: '[email protected]',
+ username: 'sjohnson',
+ provider: 'local',
+ },
+ { name: 'John Smith', email: '[email protected]', username: 'jsmith', provider: 'local' },
+ ]);
+
+ const results = await methods.searchUsers({ searchPattern: 'smith john', limit: 1 });
+
+ expect(results).toHaveLength(1);
+ expect((results[0] as unknown as t.IUser).name).toBe('John Smith');
+ });
+
test('should respect limit', async () => {
const results = await methods.searchUsers({ searchPattern: 'example', limit: 2 });
@@ -1869,9 +1898,11 @@ describe('User Methods - Database Tests', () => {
});
const results = await methods.searchUsers({ searchPattern: '.*' });
+ expect(results).toEqual([]);
- expect(results).toHaveLength(1);
- expect((results[0] as unknown as t.IUser).name).toBe('Literal .* User');
+ const literal = await methods.searchUsers({ searchPattern: 'literal' });
+ expect(literal).toHaveLength(1);
+ expect((literal[0] as unknown as t.IUser).name).toBe('Literal .* User');
});
test('should handle invalid regex syntax as literal search text', async () => {
diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts
index cc95028344..94aaddb6f1 100644
--- a/packages/data-schemas/src/methods/user.ts
+++ b/packages/data-schemas/src/methods/user.ts
@@ -16,8 +16,8 @@ import type {
} from '~/types';
import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types';
import type { CacheStore } from '~/types';
+import { scoreSearchMatch, buildUserSearchFilter } from '~/utils/search';
import { evictAuthUserDocs } from '~/utils/eviction';
-import { escapeRegExp } from '~/utils/string';
import { signPayload } from '~/crypto';
import logger from '~/config/winston';
@@ -891,7 +891,7 @@ export function createUserMethods(
}
/**
- * Search for users by pattern matching on name, email, or username (case-insensitive)
+ * Search for users by word prefix of name, email, or username (see `buildUserSearchFilter`)
* @param searchPattern - The pattern to search for
* @param limit - Maximum number of results to return
* @param fieldsToSelect - The fields to include or exclude in the returned documents
@@ -975,12 +975,13 @@ export function createUserMethods(
}
const trimmedPattern = searchPattern.trim();
- const regex = new RegExp(escapeRegExp(trimmedPattern), 'i');
+ const filter = buildUserSearchFilter(trimmedPattern);
+ if (!filter) {
+ return [];
+ }
const User = mongoose.models.User;
- const query = User.find({
- $or: [{ email: regex }, { name: regex }, { username: regex }],
- }).limit(limit * 2); // Get more results to allow for relevance sorting
+ const query = User.find(filter).limit(limit * 2); // Get more results to allow for relevance sorting
if (fieldsToSelect) {
query.select(fieldsToSelect);
@@ -988,41 +989,10 @@ export function createUserMethods(
const users = await query.lean<IUser[]>();
- // Score results by relevance
- const startsWithPattern = trimmedPattern.toLowerCase();
-
- const scoredUsers = users.map((user) => {
- const searchableFields = [user.name, user.email, user.username].filter(
- (field): field is string => typeof field === 'string' && field.length > 0,
- );
- let maxScore = 0;
-
- for (const field of searchableFields) {
- const fieldLower = field.toLowerCase();
- let score = 0;
-
- // Exact match gets highest score
- if (fieldLower === startsWithPattern) {
- score = 100;
- }
- // Starts with query gets high score
- else if (fieldLower.startsWith(startsWithPattern)) {
- score = 80;
- }
- // Contains query gets medium score
- else if (fieldLower.includes(startsWithPattern)) {
- score = 50;
- }
- // Default score for database match
- else {
- score = 10;
- }
-
- maxScore = Math.max(maxScore, score);
- }
-
- return { ...user, _searchScore: maxScore };
- });
+ const scoredUsers = users.map((user) => ({
+ ...user,
+ _searchScore: scoreSearchMatch([user.name, user.email, user.username], trimmedPattern),
+ }));
/** Top results sorted by relevance */
return scoredUsers
diff --git a/packages/data-schemas/src/methods/userGroup.spec.ts b/packages/data-schemas/src/methods/userGroup.spec.ts
index db3212e596..9c4a3a8ce0 100644
--- a/packages/data-schemas/src/methods/userGroup.spec.ts
+++ b/packages/data-schemas/src/methods/userGroup.spec.ts
@@ -117,16 +117,18 @@ describe('userGroup methods', () => {
expect(results[0].name).toBe('Design');
});
- it('matches on description field', async () => {
+ it('does not match on the description field (search covers name and email)', async () => {
const results = await methods.findGroupsByNamePattern('Eng team');
- expect(results).toHaveLength(1);
- expect(results[0].name).toBe('Engineering');
+ expect(results).toEqual([]);
});
it('treats regex metacharacters as literal text', async () => {
const results = await methods.findGroupsByNamePattern('.*');
- expect(results).toHaveLength(1);
- expect(results[0].name).toBe('Literal .* Group');
+ expect(results).toEqual([]);
+
+ const literal = await methods.findGroupsByNamePattern('literal');
+ expect(literal).toHaveLength(1);
+ expect(literal[0].name).toBe('Literal .* Group');
});
it('filters by source when provided', async () => {
@@ -1245,6 +1247,14 @@ describe('userGroup methods', () => {
expect(score).toBe(50);
});
+ it('scores accent-folded matches like their plain spelling', () => {
+ const score = methods.calculateRelevanceScore(
+ { type: PrincipalType.USER, name: 'Zoë', source: 'local' },
+ 'zoe',
+ );
+ expect(score).toBe(100);
+ });
+
it('returns 10 (default) when no substring or exact match', () => {
const score = methods.calculateRelevanceScore(
{ type: PrincipalType.USER, name: 'bob', source: 'local' },
@@ -1267,17 +1277,51 @@ describe('userGroup methods', () => {
expect(score).toBe(100);
});
- it('checks description for GROUP type', () => {
- const score = methods.calculateRelevanceScore(
+ it('ignores description for GROUP type, which search no longer matches', () => {
+ const weak = methods.calculateRelevanceScore(
{
type: PrincipalType.GROUP,
- name: 'other',
- description: 'alice team',
+ name: 'Corporate Platform',
+ description: 'plat',
source: 'local',
},
- 'alice',
+ 'plat',
);
- expect(score).toBe(80);
+ const strong = methods.calculateRelevanceScore(
+ { type: PrincipalType.GROUP, name: 'Platform Team', source: 'local' },
+ 'plat',
+ );
+ expect(weak).toBe(50);
+ expect(strong).toBeGreaterThan(weak);
+ });
+
+ it('scores a name equal to the query up to separators as an exact match', () => {
+ const score = methods.calculateRelevanceScore(
+ { type: PrincipalType.USER, name: 'Mary-Jane', source: 'local' },
+ 'mary jane',
+ );
+ expect(score).toBe(100);
+ });
+
+ it('scores a reordered full-name query as exact, above a longer prefix match', () => {
+ const exact = methods.calculateRelevanceScore(
+ { type: PrincipalType.USER, name: 'John Smith', source: 'local' },
+ 'smith john',
+ );
+ const prefix = methods.calculateRelevanceScore(
+ { type: PrincipalType.USER, name: 'Smith Johnson', source: 'local' },
+ 'smith john',
+ );
+ expect(exact).toBe(100);
+ expect(prefix).toBe(80);
+ });
+
+ it('scores query words matched across fields like a contains match', () => {
+ const score = methods.calculateRelevanceScore(
+ { type: PrincipalType.USER, name: 'John Doe', email: '[email protected]', source: 'local' },
+ 'john gmail',
+ );
+ expect(score).toBe(50);
});
it('picks the highest score across multiple fields', () => {
@@ -1402,9 +1446,11 @@ describe('userGroup methods', () => {
});
const results = await methods.searchPrincipals('.*', 10, [PrincipalType.USER]);
+ expect(results).toEqual([]);
- expect(results).toHaveLength(1);
- expect(results[0].name).toBe('Literal .* User');
+ const literal = await methods.searchPrincipals('literal', 10, [PrincipalType.USER]);
+ expect(literal).toHaveLength(1);
+ expect(literal[0].name).toBe('Literal .* User');
});
it('handles invalid regex syntax as literal search text', async () => {
diff --git a/packages/data-schemas/src/methods/userGroup.ts b/packages/data-schemas/src/methods/userGroup.ts
index 451f8e97c4..feefa83460 100644
--- a/packages/data-schemas/src/methods/userGroup.ts
+++ b/packages/data-schemas/src/methods/userGroup.ts
@@ -4,6 +4,7 @@ import { CacheKeys, PrincipalType, SystemRoles } from 'librechat-data-provider';
import type { TPrincipalSearchResult } from 'librechat-data-provider';
import type { Model, ClientSession, FilterQuery } from 'mongoose';
import type { CacheStore, IGroup, IRole, IUser } from '~/types';
+import { buildUserSearchFilter, scoreSearchMatch, buildGroupSearchFilter } from '~/utils/search';
import { isValidObjectIdString } from '~/utils/objectId';
import { scopedCacheKey } from '~/config/tenantContext';
import { escapeRegExp } from '~/utils/string';
@@ -607,7 +608,7 @@ export function createUserGroupMethods(
}
/**
- * Find groups by name pattern (case-insensitive partial match)
+ * Find groups by word prefix of name or email (see `buildGroupSearchFilter`)
* @param namePattern - The name pattern to search for
* @param source - Optional source filter ('entra', 'local', or null for all)
* @param limit - Maximum number of results to return
@@ -621,10 +622,11 @@ export function createUserGroupMethods(
session?: ClientSession,
): Promise<IGroup[]> {
const Group = mongoose.models.Group as Model<IGroup>;
- const regex = new RegExp(escapeRegExp(namePattern), 'i');
- const query: Record<string, unknown> = {
- $or: [{ name: regex }, { email: regex }, { description: regex }],
- };
+ const filter = buildGroupSearchFilter(namePattern);
+ if (!filter) {
+ return [];
+ }
+ const query: Record<string, unknown> = { ...filter };
if (source) {
query.source = source;
@@ -869,39 +871,12 @@ export function createUserGroupMethods(
* @returns Relevance score (0-100)
*/
function calculateRelevanceScore(item: TPrincipalSearchResult, searchPattern: string): number {
- const normalizedPattern = searchPattern.toLowerCase();
-
- /** Get searchable text based on type */
- const searchableFields =
+ /** The fields each search matches on; a group's description is not one of them */
+ const values =
item.type === PrincipalType.USER
- ? [item.name, item.email, item.username].filter(Boolean)
- : [item.name, item.email, item.description].filter(Boolean);
-
- let maxScore = 0;
-
- for (const field of searchableFields) {
- if (!field) continue;
- const fieldLower = field.toLowerCase();
- let score = 0;
-
- /** Exact match gets highest score */
- if (fieldLower === normalizedPattern) {
- score = 100;
- } else if (fieldLower.startsWith(normalizedPattern)) {
- /** Starts with query gets high score */
- score = 80;
- } else if (fieldLower.includes(normalizedPattern)) {
- /** Contains query gets medium score */
- score = 50;
- } else {
- /** Default score for database match */
- score = 10;
- }
-
- maxScore = Math.max(maxScore, score);
- }
-
- return maxScore;
+ ? [item.name, item.email, item.username]
+ : [item.name, item.email];
+ return scoreSearchMatch(values, searchPattern);
}
/**
@@ -970,7 +945,8 @@ export function createUserGroupMethods(
}
/**
- * Search for principals (users and groups) by pattern matching on name/email
+ * Search for principals: users and groups by word prefix (see `buildSearchTokenFilter`),
+ * roles by name
* Returns combined results in TPrincipalSearchResult format without sorting
* @param searchPattern - The pattern to search for
* @param limitPerType - Maximum number of results to return
@@ -992,17 +968,11 @@ export function createUserGroupMethods(
const escapedPattern = escapeRegExp(trimmedPattern);
const promises: Promise<TPrincipalSearchResult[]>[] = [];
- if (!typeFilter || typeFilter.includes(PrincipalType.USER)) {
- /** Note: searchUsers is imported from ~/models and needs to be passed in or implemented */
+ const userFilter = buildUserSearchFilter(trimmedPattern);
+ if (userFilter && (!typeFilter || typeFilter.includes(PrincipalType.USER))) {
const userFields = 'name email username avatar provider idOnTheSource role';
- /** For now, we'll use a direct query instead of searchUsers */
const User = mongoose.models.User as Model<IUser>;
- const regex = new RegExp(escapedPattern, 'i');
- const userQuery = User.find({
- $or: [{ name: regex }, { email: regex }, { username: regex }],
- })
- .select(userFields)
- .limit(limitPerType);
+ const userQuery = User.find(userFilter).select(userFields).limit(limitPerType);
if (session) {
userQuery.session(session);
diff --git a/packages/data-schemas/src/migrations/index.ts b/packages/data-schemas/src/migrations/index.ts
index c3ba619181..ed2699d8a4 100644
--- a/packages/data-schemas/src/migrations/index.ts
+++ b/packages/data-schemas/src/migrations/index.ts
@@ -2,3 +2,5 @@ export { dropSupersededTenantIndexes, migrateTenantIndexes } from './tenantIndex
export { dropSupersededPromptGroupIndexes } from './promptGroupIndexes';
export { createMCPAuthorityLookupIndexes } from './mcpAuthorityIndexes';
export { MCPServerNameMigrationError, backfillMCPServerNormalizedNames } from './mcpServerNames';
+export { backfillSearchTokens, warnOnMissingSearchTokens } from './searchTokens';
+export type { SearchTokenBackfillResult } from './searchTokens';
diff --git a/packages/data-schemas/src/migrations/searchTokens.spec.ts b/packages/data-schemas/src/migrations/searchTokens.spec.ts
new file mode 100644
index 0000000000..38c62f81f5
--- /dev/null
+++ b/packages/data-schemas/src/migrations/searchTokens.spec.ts
@@ -0,0 +1,247 @@
+import mongoose from 'mongoose';
+import { Collection } from 'mongodb';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import type { Document, FindOptions } from 'mongodb';
+import {
+ searchTokenIndexes,
+ USER_SEARCH_TOKEN_FIELDS,
+ GROUP_SEARCH_TOKEN_FIELDS,
+} from '~/utils/search';
+import { backfillSearchTokens, warnOnMissingSearchTokens } from './searchTokens';
+import logger from '~/config/winston';
+
+jest.mock('~/config/winston', () => ({
+ error: jest.fn(),
+ warn: jest.fn(),
+ info: jest.fn(),
+ debug: jest.fn(),
+}));
+
+let mongoServer: MongoMemoryServer;
+
+beforeAll(async () => {
+ mongoServer = await MongoMemoryServer.create();
+ await mongoose.connect(mongoServer.getUri());
+});
+
+afterAll(async () => {
+ await mongoose.disconnect();
+ await mongoServer.stop();
+});
+
+beforeEach(async () => {
+ await mongoose.connection.dropDatabase();
+ jest.clearAllMocks();
+});
+
+const users = () => mongoose.connection.db!.collection('users');
+const groups = () => mongoose.connection.db!.collection('groups');
+
+describe('backfillSearchTokens', () => {
+ it('backfills users and groups across tenants in batches and is idempotent', async () => {
+ await users().insertMany([
+ { name: 'Ana Lima', email: '[email protected]', username: 'ana', tenantId: 'tenant-a' },
+ { name: 'Ben Ode', email: '[email protected]', tenantId: 'tenant-b' },
+ { email: '[email protected]' },
+ {
+ name: 'Already Done',
+ email: '[email protected]',
+ nameTokens: ['already', 'done'],
+ emailTokens: ['[email protected]', 'done', 'z', 'io'],
+ usernameTokens: [],
+ },
+ ]);
+ await groups().insertMany([{ name: 'Core Team', source: 'local' }]);
+
+ const result = await backfillSearchTokens(mongoose.connection, { batchSize: 2 });
+ expect(result).toEqual({
+ pending: { users: 3, groups: 1 },
+ updated: { users: 3, groups: 1 },
+ });
+
+ const ana = await users().findOne({ email: '[email protected]' });
+ expect(ana).toMatchObject({
+ tenantId: 'tenant-a',
+ nameTokens: ['ana', 'lima'],
+ emailTokens: ['[email protected]', 'ana', 'x', 'io'],
+ usernameTokens: ['ana'],
+ });
+ expect(await users().findOne({ email: '[email protected]' })).toMatchObject({
+ nameTokens: [],
+ usernameTokens: [],
+ });
+ expect(await groups().findOne({ name: 'Core Team' })).toMatchObject({
+ nameTokens: ['core', 'team'],
+ emailTokens: [],
+ });
+
+ await expect(backfillSearchTokens(mongoose.connection)).resolves.toEqual({
+ pending: { users: 0, groups: 0 },
+ updated: { users: 0, groups: 0 },
+ });
+ });
+
+ it('completes documents that a partial update tokenized for only one field', async () => {
+ await users().insertOne({ name: 'Part Ial', email: '[email protected]', emailTokens: ['[email protected]'] });
+ await backfillSearchTokens(mongoose.connection);
+ expect(await users().findOne({ email: '[email protected]' })).toMatchObject({
+ nameTokens: ['part', 'ial'],
+ emailTokens: ['[email protected]', 'p', 'x', 'io'],
+ });
+ });
+
+ it('repairs tokens left stale by a writer that did not maintain them', async () => {
+ await users().insertOne({
+ name: 'New Name',
+ email: '[email protected]',
+ nameTokens: ['old', 'name'],
+ emailTokens: ['[email protected]', 'stale', 'x', 'io'],
+ usernameTokens: [],
+ });
+ await expect(backfillSearchTokens(mongoose.connection, { dryRun: true })).resolves.toEqual({
+ pending: { users: 1, groups: 0 },
+ updated: { users: 0, groups: 0 },
+ });
+ await expect(backfillSearchTokens(mongoose.connection)).resolves.toEqual({
+ pending: { users: 1, groups: 0 },
+ updated: { users: 1, groups: 0 },
+ });
+ expect(await users().findOne({ email: '[email protected]' })).toMatchObject({
+ nameTokens: ['new', 'name'],
+ });
+ });
+
+ it('creates the token indexes when the schema indexes were never built', async () => {
+ await users().insertOne({ name: 'No Index', email: '[email protected]' });
+ await groups().insertOne({ name: 'Plain Group' });
+ await backfillSearchTokens(mongoose.connection);
+ const userIndexes = (await users().indexes()).map((index) => index.name);
+ const groupIndexes = (await groups().indexes()).map((index) => index.name);
+ expect(userIndexes).toEqual(
+ expect.arrayContaining([
+ 'nameTokens_1_tenantId_1',
+ 'emailTokens_1_tenantId_1',
+ 'usernameTokens_1_tenantId_1',
+ 'tenantId_1_nameTokens_1',
+ 'tenantId_1_emailTokens_1',
+ 'tenantId_1_usernameTokens_1',
+ ]),
+ );
+ expect(groupIndexes).toEqual(
+ expect.arrayContaining([
+ 'nameTokens_1_tenantId_1',
+ 'emailTokens_1_tenantId_1',
+ 'tenantId_1_nameTokens_1',
+ 'tenantId_1_emailTokens_1',
+ ]),
+ );
+ });
+
+ it('writes nothing on a dry run', async () => {
+ await users().insertOne({ name: 'Dry Run', email: '[email protected]' });
+ await expect(backfillSearchTokens(mongoose.connection, { dryRun: true })).resolves.toEqual({
+ pending: { users: 1, groups: 0 },
+ updated: { users: 0, groups: 0 },
+ });
+ expect(await users().findOne({ email: '[email protected]' })).not.toHaveProperty('nameTokens');
+ });
+});
+
+describe('warnOnMissingSearchTokens', () => {
+ it('warns only while documents lack tokens', async () => {
+ await users().insertOne({ name: 'Old User', email: '[email protected]' });
+ await groups().insertOne({ name: 'Old Group' });
+ await warnOnMissingSearchTokens(mongoose.connection);
+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users and groups'));
+
+ jest.clearAllMocks();
+ await backfillSearchTokens(mongoose.connection);
+ await warnOnMissingSearchTokens(mongoose.connection);
+ expect(logger.warn).not.toHaveBeenCalled();
+ });
+
+ /** Runs the startup check and explains the `users` probe it actually issued. */
+ const explainUserProbe = async () => {
+ const findOne = jest.spyOn(Collection.prototype, 'findOne');
+ try {
+ await warnOnMissingSearchTokens(mongoose.connection);
+ const call = findOne.mock.contexts.findIndex(
+ (collection) => collection.collectionName === 'users',
+ );
+ const [filter, options] = findOne.mock.calls[call] as [Document, FindOptions];
+ expect(options.maxTimeMS).toBeGreaterThan(0);
+ return await users().find(filter).limit(1).explain('executionStats');
+ } finally {
+ findOne.mockRestore();
+ }
+ };
+
+ it('warns without probing documents while the token indexes do not exist', async () => {
+ await users().insertMany(
+ Array.from({ length: 200 }, (_, i) => ({ name: `Old ${i}`, email: `old${i}@x.io` })),
+ );
+ const findOne = jest.spyOn(Collection.prototype, 'findOne');
+ try {
+ await warnOnMissingSearchTokens(mongoose.connection);
+ const probed = findOne.mock.contexts.some(
+ (collection) => collection.collectionName === 'users',
+ );
+ expect(probed).toBe(false);
+ } finally {
+ findOne.mockRestore();
+ }
+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users'));
+ });
+
+ it('examines no documents once the backfill has built the indexes and filled every token', async () => {
+ await users().insertMany(
+ Array.from({ length: 50 }, (_, i) => ({ name: `User ${i}`, email: `u${i}@x.io` })),
+ );
+ await backfillSearchTokens(mongoose.connection);
+ jest.clearAllMocks();
+ const explain = await explainUserProbe();
+ expect(JSON.stringify(explain.queryPlanner.winningPlan)).not.toContain('COLLSCAN');
+ expect(explain.executionStats.totalDocsExamined).toBe(0);
+ expect(logger.warn).not.toHaveBeenCalled();
+ });
+
+ it('warns when every document has tokens but the token indexes were never built', async () => {
+ await users().insertOne({
+ name: 'Fresh User',
+ email: '[email protected]',
+ nameTokens: ['fresh', 'user'],
+ emailTokens: ['[email protected]', 'fresh', 'x', 'io'],
+ usernameTokens: [],
+ });
+ await warnOnMissingSearchTokens(mongoose.connection);
+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users'));
+
+ jest.clearAllMocks();
+ await backfillSearchTokens(mongoose.connection);
+ await warnOnMissingSearchTokens(mongoose.connection);
+ expect(logger.warn).not.toHaveBeenCalled();
+ });
+
+ it('stays quiet on a database with no users or groups yet', async () => {
+ await warnOnMissingSearchTokens(mongoose.connection);
+ expect(logger.warn).not.toHaveBeenCalled();
+ expect(logger.error).not.toHaveBeenCalled();
+ });
+
+ it('logs a failed probe instead of failing startup', async () => {
+ /** Every token index exists, so the check reaches the probe, which then fails. */
+ const indexes = [...USER_SEARCH_TOKEN_FIELDS, ...GROUP_SEARCH_TOKEN_FIELDS]
+ .flatMap(searchTokenIndexes)
+ .map((key) => ({ key }));
+ const findOne = jest.fn(() => Promise.reject(new Error('down')));
+ const broken = {
+ db: { collection: () => ({ indexes: async () => indexes, findOne }) },
+ };
+ await expect(
+ warnOnMissingSearchTokens(broken as unknown as typeof mongoose.connection),
+ ).resolves.toBeUndefined();
+ expect(findOne).toHaveBeenCalled();
+ expect(logger.error).toHaveBeenCalled();
+ expect(logger.warn).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/data-schemas/src/migrations/searchTokens.ts b/packages/data-schemas/src/migrations/searchTokens.ts
new file mode 100644
index 0000000000..42d668883a
--- /dev/null
+++ b/packages/data-schemas/src/migrations/searchTokens.ts
@@ -0,0 +1,178 @@
+import type { AnyBulkWriteOperation, Document } from 'mongodb';
+import type { Connection } from 'mongoose';
+import type { SearchTokenField } from '~/utils/search';
+import {
+ searchTokenIndexes,
+ computeSearchTokenSet,
+ USER_SEARCH_TOKEN_FIELDS,
+ GROUP_SEARCH_TOKEN_FIELDS,
+} from '~/utils/search';
+import { buildIndexWithRetry } from '~/utils/retry';
+import logger from '~/config/winston';
+
+const DEFAULT_BATCH_SIZE = 500;
+/** Upper bound on the startup probe, in case it runs before the token indexes exist. */
+const STARTUP_PROBE_MAX_TIME_MS = 5_000;
+
+const SEARCH_TOKEN_COLLECTIONS: ReadonlyArray<{
+ name: string;
+ fields: readonly SearchTokenField[];
+}> = [
+ { name: 'users', fields: USER_SEARCH_TOKEN_FIELDS },
+ { name: 'groups', fields: GROUP_SEARCH_TOKEN_FIELDS },
+];
+
+export interface SearchTokenBackfillResult {
+ /** Documents whose token fields are missing or no longer match their source, per collection. */
+ pending: Record<string, number>;
+ /** Documents written, per collection; zero on a dry run. */
+ updated: Record<string, number>;
+}
+
+/** Index-bound: each branch seeks one token index on its missing-key bounds. */
+function missingTokens(fields: readonly SearchTokenField[]): Document {
+ return { $or: fields.map((field) => ({ [field.tokens]: { $exists: false } })) };
+}
+
+function sameTokens(stored: unknown, expected: string[]): boolean {
+ return (
+ Array.isArray(stored) &&
+ stored.length === expected.length &&
+ stored.every((token, index) => token === expected[index])
+ );
+}
+
+/**
+ * Brings the search-token arrays on users and groups in line with their source
+ * fields: fills documents saved before the tokens existed and repairs tokens
+ * left stale by a writer that did not maintain them (an older server during a
+ * rolling deploy, a raw collection write). Every document is read and only the
+ * ones whose tokens differ are written, so it is idempotent and resumable; run
+ * it again once a rollout completes. Each write is guarded by the source values
+ * it was computed from, so a concurrent rename (whose own update already wrote
+ * fresh tokens) is skipped. Runs across all tenants on the raw collections; the
+ * `_id` filter keeps every write on its own document, and `tenantId` is never
+ * written.
+ */
+export async function backfillSearchTokens(
+ connection: Connection,
+ options: { batchSize?: number; dryRun?: boolean } = {},
+): Promise<SearchTokenBackfillResult> {
+ const batchSize = Math.max(1, options.batchSize ?? DEFAULT_BATCH_SIZE);
+ const result: SearchTokenBackfillResult = { pending: {}, updated: {} };
+
+ for (const { name, fields } of SEARCH_TOKEN_COLLECTIONS) {
+ const collection = connection.db!.collection(name);
+ if (!options.dryRun) {
+ /** Deployments running with `MONGO_AUTO_INDEX` off never build schema indexes, and
+ * without these the token filters would scan the collection. Same specs and default
+ * names as the schema declaration, so this is a no-op where Mongoose already built them. */
+ for (const field of fields) {
+ for (const index of searchTokenIndexes(field)) {
+ await buildIndexWithRetry(
+ () => collection.createIndex(index),
+ `createIndex(${name}.${Object.keys(index).join('_')})`,
+ );
+ }
+ }
+ }
+ result.pending[name] = 0;
+ result.updated[name] = 0;
+
+ const projection = Object.fromEntries(
+ fields.flatMap((field) => [
+ [field.source, 1],
+ [field.tokens, 1],
+ ]),
+ );
+ let batch: AnyBulkWriteOperation[] = [];
+ const flush = async () => {
+ if (batch.length === 0) {
+ return;
+ }
+ // eslint-disable-next-line no-restricted-syntax -- offline all-tenant migration; `_id` filters keep each write on its own document
+ const written = await collection.bulkWrite(batch, { ordered: false });
+ result.updated[name] += written.modifiedCount;
+ batch = [];
+ };
+
+ for await (const doc of collection.find({}, { projection })) {
+ const expected = computeSearchTokenSet(fields, doc);
+ if (fields.every((field) => sameTokens(doc[field.tokens], expected[field.tokens]))) {
+ continue;
+ }
+ result.pending[name] += 1;
+ if (options.dryRun) {
+ continue;
+ }
+ const guard = Object.fromEntries(
+ fields.map((field) => [field.source, doc[field.source] ?? null]),
+ );
+ batch.push({ updateOne: { filter: { _id: doc._id, ...guard }, update: { $set: expected } } });
+ if (batch.length >= batchSize) {
+ await flush();
+ }
+ }
+ await flush();
+ if (!options.dryRun) {
+ logger.info(
+ `[SearchTokenMigration] ${name}: ${result.updated[name]} of ${result.pending[name]} documents updated`,
+ );
+ }
+ }
+ return result;
+}
+
+/**
+ * Logs a startup warning when users or groups still lack search tokens. Those
+ * documents stay findable through the slower unindexed fallback until
+ * `npm run migrate:search-tokens` runs. A best-effort diagnostic: a failed
+ * check is logged and never blocks startup.
+ *
+ * Also warns when the token indexes are missing: with `MONGO_AUTO_INDEX` off,
+ * a database whose documents all received tokens from the schema middleware
+ * still never builds them until the backfill runs.
+ *
+ * The indexes are checked first, so the probe for a token-less document only
+ * runs once they exist and it can seek them; `maxTimeMS` bounds it regardless.
+ */
+export async function warnOnMissingSearchTokens(connection: Connection): Promise<void> {
+ try {
+ const pending = await Promise.all(
+ SEARCH_TOKEN_COLLECTIONS.map(async ({ name, fields }) => {
+ const collection = connection.db!.collection(name);
+ const indexes = await collection.indexes().catch((error: { codeName?: string }) => {
+ /** No collection yet: nothing to search, nothing to migrate. */
+ if (error?.codeName === 'NamespaceNotFound') {
+ return null;
+ }
+ throw error;
+ });
+ if (indexes == null) {
+ return null;
+ }
+ const built = new Set(indexes.map((index) => JSON.stringify(index.key)));
+ const unindexed = fields.some((field) =>
+ searchTokenIndexes(field).some((index) => !built.has(JSON.stringify(index))),
+ );
+ /** Without the indexes the probe below would scan; the warning is already due. */
+ if (unindexed) {
+ return name;
+ }
+ const unmigrated = await collection.findOne(missingTokens(fields), {
+ projection: { _id: 1 },
+ maxTimeMS: STARTUP_PROBE_MAX_TIME_MS,
+ });
+ return unmigrated != null ? name : null;
+ }),
+ );
+ const names = pending.filter((name): name is string => name != null);
+ if (names.length > 0) {
+ logger.warn(
+ `[SearchTokenMigration] Some ${names.join(' and ')} lack search tokens or their indexes; people search scans the collection until you run: npm run migrate:search-tokens`,
+ );
+ }
+ } catch (error) {
+ logger.error('[SearchTokenMigration] Failed to check search token migration:', error);
+ }
+}
diff --git a/packages/data-schemas/src/models/plugins/searchTokens.spec.ts b/packages/data-schemas/src/models/plugins/searchTokens.spec.ts
new file mode 100644
index 0000000000..edd1c6154d
--- /dev/null
+++ b/packages/data-schemas/src/models/plugins/searchTokens.spec.ts
@@ -0,0 +1,318 @@
+import mongoose from 'mongoose';
+import { PrincipalType } from 'librechat-data-provider';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import type * as t from '~/types';
+import { createUserGroupMethods } from '~/methods/userGroup';
+import { tenantStorage } from '~/config/tenantContext';
+import { buildUserSearchFilter } from '~/utils/search';
+import { createModels } from '~/models';
+
+jest.mock('~/config/winston', () => ({
+ error: jest.fn(),
+ warn: jest.fn(),
+ info: jest.fn(),
+ debug: jest.fn(),
+}));
+
+const TOKENS = '+nameTokens +emailTokens +usernameTokens';
+
+let mongoServer: MongoMemoryServer;
+let User: mongoose.Model<t.IUser>;
+let Group: mongoose.Model<t.IGroup>;
+let methods: ReturnType<typeof createUserGroupMethods>;
+
+beforeAll(async () => {
+ mongoServer = await MongoMemoryServer.create();
+ await mongoose.connect(mongoServer.getUri());
+ const models = createModels(mongoose);
+ User = models.User;
+ Group = models.Group as mongoose.Model<t.IGroup>;
+ methods = createUserGroupMethods(mongoose);
+});
+
+afterAll(async () => {
+ await mongoose.disconnect();
+ await mongoServer.stop();
+});
+
+beforeEach(async () => {
+ await mongoose.connection.dropDatabase();
+});
+
+const user = (name: string, email: string, username = '', extra: Partial<t.IUser> = {}) => ({
+ name,
+ email,
+ username,
+ provider: 'local',
+ ...extra,
+});
+
+async function tokensOf(id: unknown) {
+ return User.findById(id).select(TOKENS).lean<t.IUser>();
+}
+
+async function userNames(query: string, limit = 10): Promise<string[]> {
+ const results = await methods.searchPrincipals(query, limit, [PrincipalType.USER]);
+ return results.map((r) => r.name ?? '').sort();
+}
+
+describe('search token write paths', () => {
+ it('derives tokens on create and on save of a changed name', async () => {
+ const doc = await User.create(user('José Álvarez', '[email protected]', 'jalvarez'));
+ expect(await tokensOf(doc._id)).toMatchObject({
+ nameTokens: ['jose', 'alvarez'],
+ emailTokens: ['[email protected]', 'jose', 'alvarez', 'example', 'org'],
+ usernameTokens: ['jalvarez'],
+ });
+
+ doc.name = 'Joe Bloggs';
+ await doc.save();
+ const saved = await tokensOf(doc._id);
+ expect(saved?.nameTokens).toEqual(['joe', 'bloggs']);
+ expect(saved?.usernameTokens).toEqual(['jalvarez']);
+ });
+
+ it('updates tokens through findOneAndUpdate, findByIdAndUpdate, updateOne and updateMany', async () => {
+ const [a, b] = await User.create([user('Ann A', '[email protected]'), user('Ben B', '[email protected]')]);
+
+ await User.findOneAndUpdate({ _id: a._id }, { $set: { name: 'Carla Diaz' } });
+ expect((await tokensOf(a._id))?.nameTokens).toEqual(['carla', 'diaz']);
+
+ await User.findByIdAndUpdate(a._id, { email: '[email protected]' });
+ expect((await tokensOf(a._id))?.emailTokens).toEqual(['[email protected]', 'carla', 'new', 'dev']);
+
+ await User.updateOne({ _id: b._id }, { $set: { username: 'ben_b' } });
+ expect((await tokensOf(b._id))?.usernameTokens).toEqual(['ben_b', 'ben', 'b']);
+
+ await User.updateMany({}, { $unset: { name: '' } });
+ expect((await tokensOf(a._id))?.nameTokens).toEqual([]);
+ expect((await tokensOf(b._id))?.nameTokens).toEqual([]);
+ });
+
+ it('sets tokens on upsert inserts and insertMany', async () => {
+ await User.updateOne(
+ { email: '[email protected]' },
+ { $setOnInsert: { name: 'New Person', email: '[email protected]', provider: 'local' } },
+ { upsert: true },
+ );
+ const upserted = await User.findOne({ email: '[email protected]' }).select(TOKENS).lean<t.IUser>();
+ expect(upserted?.nameTokens).toEqual(['new', 'person']);
+ expect(upserted?.usernameTokens).toEqual([]);
+
+ await User.updateOne(
+ { email: '[email protected]' },
+ { $set: { name: 'From Filter', provider: 'local' } },
+ { upsert: true },
+ );
+ const fromFilter = await User.findOne({ email: '[email protected]' }).select(TOKENS).lean<t.IUser>();
+ expect(fromFilter?.emailTokens).toEqual(['[email protected]', 'filter', 'x', 'io']);
+
+ await User.insertMany([user('Many One', '[email protected]', 'm1')]);
+ const inserted = await User.findOne({ email: '[email protected]' }).select(TOKENS).lean<t.IUser>();
+ expect(inserted?.nameTokens).toEqual(['many', 'one']);
+ expect(inserted?.usernameTokens).toEqual(['m1']);
+ });
+
+ it('keeps tokens out of default reads', async () => {
+ const doc = await User.create(user('Hidden Tokens', '[email protected]'));
+ const plain = await User.findById(doc._id).lean<t.IUser>();
+ expect(plain).not.toHaveProperty('nameTokens');
+ });
+
+ it('updates group tokens through create, upsertGroupByExternalId and updateGroupById', async () => {
+ const group = await methods.createGroup({ name: 'Core Platform', source: 'local' });
+ const read = () =>
+ Group.findById(group._id).select('+nameTokens +emailTokens').lean<t.IGroup>();
+ expect((await read())?.nameTokens).toEqual(['core', 'platform']);
+
+ await methods.updateGroupById(group._id, { name: 'Data Science', email: '[email protected]' });
+ expect(await read()).toMatchObject({
+ nameTokens: ['data', 'science'],
+ emailTokens: ['[email protected]', 'ds', 'corp', 'io'],
+ });
+
+ const entra = await methods.upsertGroupByExternalId('ext-1', 'entra', { name: 'Sales EMEA' });
+ const entraTokens = await Group.findById(entra?._id).select('+nameTokens').lean<t.IGroup>();
+ expect(entraTokens?.nameTokens).toEqual(['sales', 'emea']);
+ expect(await Group.countDocuments({ _id: entra?._id, emailTokens: { $exists: true } })).toBe(1);
+ });
+
+ it('keeps tokens when an update assigns undefined to a source field', async () => {
+ const doc = await User.create(user('Kept Name', '[email protected]'));
+ await User.findByIdAndUpdate(doc._id, { $set: { name: undefined, role: 'ADMIN' } });
+ const stored = await User.findById(doc._id).select(`name ${TOKENS}`).lean<t.IUser>();
+ expect(stored?.name).toBe('Kept Name');
+ expect(stored?.nameTokens).toEqual(['kept', 'name']);
+ });
+});
+
+describe('searchPrincipals word-prefix search', () => {
+ beforeEach(async () => {
+ await User.create([
+ user('John Smith', '[email protected]', 'jsmith'),
+ user('Johanna Müller', '[email protected]', 'hanna_m'),
+ user('Bob Johnson', '[email protected]', 'bobby'),
+ user('Zoë Ng', '[email protected]', 'zng'),
+ ]);
+ });
+
+ it('matches the prefix of a first name, a last name, an email word, a domain and a username', async () => {
+ expect(await userNames('joh')).toEqual(['Bob Johnson', 'Johanna Müller', 'John Smith']);
+ expect(await userNames('smi')).toEqual(['John Smith']);
+ expect(await userNames('mull')).toEqual(['Johanna Müller']);
+ expect(await userNames('acme')).toEqual(['John Smith', 'Zoë Ng']);
+ expect(await userNames('hanna')).toEqual(['Johanna Müller']);
+ expect(await userNames('john.smith@ac')).toEqual(['John Smith']);
+ });
+
+ it('requires every query token, across fields', async () => {
+ expect(await userNames('john smi')).toEqual(['John Smith']);
+ expect(await userNames('smith john')).toEqual(['John Smith']);
+ expect(await userNames('jo acme')).toEqual(['John Smith']);
+ expect(await userNames('john globex')).toEqual([]);
+ });
+
+ it('is case- and accent-insensitive', async () => {
+ expect(await userNames('ZOE')).toEqual(['Zoë Ng']);
+ expect(await userNames('MÜLLER')).toEqual(['Johanna Müller']);
+ });
+
+ it('does not match inside a word (intended: only word prefixes are index-bound)', async () => {
+ expect(await userNames('ohn')).toEqual([]);
+ expect(await userNames('mith')).toEqual([]);
+ });
+
+ it('respects the limit', async () => {
+ expect(await userNames('jo', 2)).toHaveLength(2);
+ });
+
+ it('finds documents written before tokens existed through the substring fallback', async () => {
+ await User.collection.insertOne({ name: 'Legacy Person', email: '[email protected]' });
+ expect(await userNames('gacy')).toEqual(['Legacy Person']);
+ });
+
+ it('stays inside the active tenant', async () => {
+ await tenantStorage.run({ tenantId: 'tenant-a' }, () =>
+ User.create(user('John Tenant', '[email protected]')),
+ );
+ await tenantStorage.run({ tenantId: 'tenant-b' }, () =>
+ User.create(user('John Other', '[email protected]')),
+ );
+ const names = await tenantStorage.run({ tenantId: 'tenant-a' }, () => userNames('john'));
+ expect(names).toEqual(['John Tenant']);
+ });
+
+ it('finds groups by name and email word prefix', async () => {
+ await Group.create([
+ { name: 'Platform Team', source: 'local', email: '[email protected]' },
+ { name: 'Design', source: 'local', description: 'Platform design' },
+ ]);
+ const groups = await methods.searchPrincipals('plat', 10, [PrincipalType.GROUP]);
+ expect(groups.map((g) => g.name)).toEqual(['Platform Team']);
+ });
+});
+
+describe('search token serialization', () => {
+ it('keeps token fields out of the JSON and object output of a freshly created document', async () => {
+ const [group] = await Group.create([{ name: 'Platform Team', source: 'local' }]);
+ const created = await User.create(user('Ana Lima', '[email protected]', 'ana'));
+ for (const doc of [group, created]) {
+ expect(doc.get('nameTokens')).toBeDefined();
+ expect(doc.toJSON()).not.toHaveProperty('nameTokens');
+ expect(doc.toObject()).not.toHaveProperty('nameTokens');
+ expect(doc.toObject()).not.toHaveProperty('emailTokens');
+ }
+ const stored = await Group.findById(group._id).select('+nameTokens').lean();
+ expect(stored?.nameTokens).toEqual(['platform', 'team']);
+ });
+});
+
+describe('search query plan', () => {
+ const planStages = (plan: unknown): string[] => {
+ const stages: string[] = [];
+ const visit = (node: unknown) => {
+ if (node == null || typeof node !== 'object') {
+ return;
+ }
+ const record = node as Record<string, unknown>;
+ if (typeof record.stage === 'string') {
+ stages.push(record.stage);
+ }
+ for (const value of Object.values(record)) {
+ visit(value);
+ }
+ };
+ visit(plan);
+ return stages;
+ };
+
+ beforeEach(async () => {
+ await User.createIndexes();
+ await User.create([
+ user('John Smith', '[email protected]', 'jsmith'),
+ user('Bob Johnson', '[email protected]', 'bobby'),
+ ]);
+ });
+
+ it.each(['john', 'john smi', 'john.smith@acme'])(
+ 'uses index scans, never a collection scan, for "%s"',
+ async (query) => {
+ const filter = buildUserSearchFilter(query)!;
+ const explain = (await User.find(filter).limit(10).explain('queryPlanner')) as unknown as {
+ queryPlanner: { winningPlan: unknown };
+ };
+ const stages = planStages(explain.queryPlanner.winningPlan);
+ expect(stages).toContain('IXSCAN');
+ expect(stages).not.toContain('COLLSCAN');
+ },
+ );
+
+ it('uses index scans with the admin search sort by name', async () => {
+ const filter = buildUserSearchFilter('jo')!;
+ const explain = (await User.find(filter)
+ .sort({ name: 1 })
+ .limit(20)
+ .explain('queryPlanner')) as unknown as { queryPlanner: { winningPlan: unknown } };
+ const stages = planStages(explain.queryPlanner.winningPlan);
+ expect(stages).toContain('IXSCAN');
+ expect(stages).not.toContain('COLLSCAN');
+ });
+
+ it('uses index scans under a tenant scope', async () => {
+ const filter = buildUserSearchFilter('john')!;
+ const explain = (await tenantStorage.run({ tenantId: 'tenant-a' }, () =>
+ User.find(filter).limit(10).explain('queryPlanner').exec(),
+ )) as unknown as { queryPlanner: { winningPlan: unknown } };
+ const stages = planStages(explain.queryPlanner.winningPlan);
+ expect(stages).toContain('IXSCAN');
+ expect(stages).not.toContain('COLLSCAN');
+ });
+
+ it('seeks only the scoped tenant keys when every tenant shares the prefix', async () => {
+ /** The scoped tenant is large with two matches; every other tenant is all matches,
+ * so neither a tenant-only nor a token-first seek can stay small. */
+ const doc = (name: string, i: number, tenant: string) => ({
+ name: `${name} ${i}`,
+ email: `${name.toLowerCase()}${i}@${tenant}.io`,
+ tenantId: tenant,
+ nameTokens: [name.toLowerCase(), String(i)],
+ emailTokens: [
+ `${name.toLowerCase()}${i}@${tenant}.io`,
+ `${name.toLowerCase()}${i}`,
+ tenant,
+ 'io',
+ ],
+ usernameTokens: [],
+ });
+ await User.collection.insertMany([
+ ...Array.from({ length: 300 }, (_, i) => doc(i < 2 ? 'Johnny' : 'Anna', i, 'tenant-3')),
+ ...Array.from({ length: 300 }, (_, i) => doc('Johnny', i, `tenant-${10 + (i % 10)}`)),
+ ]);
+ const filter = buildUserSearchFilter('johnny')!;
+ const explain = (await tenantStorage.run({ tenantId: 'tenant-3' }, () =>
+ User.find(filter).explain('executionStats').exec(),
+ )) as unknown as { executionStats: { nReturned: number; totalKeysExamined: number } };
+ expect(explain.executionStats.nReturned).toBe(2);
+ expect(explain.executionStats.totalKeysExamined).toBeLessThanOrEqual(10);
+ });
+});
diff --git a/packages/data-schemas/src/models/plugins/searchTokens.ts b/packages/data-schemas/src/models/plugins/searchTokens.ts
new file mode 100644
index 0000000000..67b935b57b
--- /dev/null
+++ b/packages/data-schemas/src/models/plugins/searchTokens.ts
@@ -0,0 +1,71 @@
+import type { Schema, Query } from 'mongoose';
+import type { SearchTokenField } from '~/utils/search';
+import { withSearchTokens, searchTokenIndexes, computeSearchTokens } from '~/utils/search';
+
+/**
+ * Adds the derived search-token arrays of `fields` to `schema`, indexes each
+ * with `tenantId` in both key orders (see `searchTokenIndexes`), and keeps them in sync on `save`, `insertMany` and the
+ * update queries. `bulkWrite` runs no middleware: callers that change a source
+ * field through it must apply `withSearchTokens` themselves.
+ *
+ * Token fields are `select: false` and stripped from `toJSON`/`toObject`
+ * output (a freshly saved document still holds them); they exist for the
+ * index, not for readers.
+ */
+export function applySearchTokens(schema: Schema, fields: readonly SearchTokenField[]): void {
+ for (const field of fields) {
+ schema.add({ [field.tokens]: { type: [String], default: undefined, select: false } });
+ for (const index of searchTokenIndexes(field)) {
+ schema.index(index);
+ }
+ }
+
+ for (const option of ['toJSON', 'toObject'] as const) {
+ schema.set(option, {
+ ...schema.get(option),
+ transform(_doc, ret: Record<string, unknown>) {
+ for (const field of fields) {
+ delete ret[field.tokens];
+ }
+ return ret;
+ },
+ });
+ }
+
+ schema.pre('save', function () {
+ for (const field of fields) {
+ if (this.isNew || this.isModified(field.source)) {
+ this.set(field.tokens, computeSearchTokens(field.kind, this.get(field.source)));
+ }
+ }
+ });
+
+ schema.pre(
+ ['findOneAndUpdate', 'updateOne', 'updateMany'],
+ { document: false, query: true },
+ function (this: Query<unknown, unknown>) {
+ const update = this.getUpdate();
+ if (!update) {
+ return;
+ }
+ const next = withSearchTokens(fields, update, {
+ upsert: this.getOptions().upsert === true,
+ filter: this.getFilter(),
+ });
+ if (next !== update) {
+ this.setUpdate(next);
+ }
+ },
+ );
+
+ schema.pre('insertMany', function (next, docs: unknown) {
+ if (Array.isArray(docs)) {
+ for (const doc of docs as Array<Record<string, unknown>>) {
+ for (const field of fields) {
+ doc[field.tokens] = computeSearchTokens(field.kind, doc[field.source]);
+ }
+ }
+ }
+ next();
+ });
+}
diff --git a/packages/data-schemas/src/schema/group.ts b/packages/data-schemas/src/schema/group.ts
index bb19e4aa81..17c5e1bd06 100644
--- a/packages/data-schemas/src/schema/group.ts
+++ b/packages/data-schemas/src/schema/group.ts
@@ -1,5 +1,7 @@
import { Schema } from 'mongoose';
import type { IGroup } from '~/types';
+import { applySearchTokens } from '~/models/plugins/searchTokens';
+import { GROUP_SEARCH_TOKEN_FIELDS } from '~/utils/search';
const groupSchema: Schema<IGroup> = new Schema<IGroup>(
{
@@ -57,5 +59,6 @@ groupSchema.index(
},
);
groupSchema.index({ memberIds: 1, tenantId: 1 });
+applySearchTokens(groupSchema, GROUP_SEARCH_TOKEN_FIELDS);
export default groupSchema;
diff --git a/packages/data-schemas/src/schema/user.ts b/packages/data-schemas/src/schema/user.ts
index 87ba658ce0..0b7329d322 100644
--- a/packages/data-schemas/src/schema/user.ts
+++ b/packages/data-schemas/src/schema/user.ts
@@ -1,5 +1,7 @@
import { Schema } from 'mongoose';
import { SystemRoles, STATEFUL_CODE_ENVIRONMENTS } from 'librechat-data-provider';
+import { applySearchTokens } from '~/models/plugins/searchTokens';
+import { USER_SEARCH_TOKEN_FIELDS } from '~/utils/search';
import { IUser } from '~/types';
// Session sub-schema
@@ -244,6 +246,7 @@ userSchema.index({ idOnTheSource: 1, openidIssuer: 1, tenantId: 1 });
them. The rare cleanup that pulls a deleted agent from every user names the tenant too;
the one that does not (`$in` over a batch of ids) is a maintenance write, not a page. */
userSchema.index({ tenantId: 1, 'favorites.agentId': 1 });
+applySearchTokens(userSchema, USER_SEARCH_TOKEN_FIELDS);
const oAuthIdFields = [
'googleId',
diff --git a/packages/data-schemas/src/types/group.ts b/packages/data-schemas/src/types/group.ts
index 15cb6f288e..f935b9837f 100644
--- a/packages/data-schemas/src/types/group.ts
+++ b/packages/data-schemas/src/types/group.ts
@@ -15,6 +15,9 @@ export interface IGroup extends Document {
createdAt?: Date;
updatedAt?: Date;
tenantId?: string;
+ /** Derived search tokens (see `~/utils/search`); `select: false`. */
+ nameTokens?: string[];
+ emailTokens?: string[];
}
export interface CreateGroupRequest {
diff --git a/packages/data-schemas/src/types/user.ts b/packages/data-schemas/src/types/user.ts
index e2b3cd861b..9b0e8837b8 100644
--- a/packages/data-schemas/src/types/user.ts
+++ b/packages/data-schemas/src/types/user.ts
@@ -89,6 +89,10 @@ export interface IUser extends Document {
/** Field for external source identification (for consistency with TPrincipal schema) */
idOnTheSource?: string;
tenantId?: string;
+ /** Derived search tokens (see `~/utils/search`); `select: false`. */
+ nameTokens?: string[];
+ emailTokens?: string[];
+ usernameTokens?: string[];
federatedTokens?: OIDCTokens;
openidTokens?: OIDCTokens;
}
diff --git a/packages/data-schemas/src/utils/index.ts b/packages/data-schemas/src/utils/index.ts
index f5a4c6fd48..828eb80429 100644
--- a/packages/data-schemas/src/utils/index.ts
+++ b/packages/data-schemas/src/utils/index.ts
@@ -1,5 +1,6 @@
export * from './principal';
export * from './string';
+export * from './search';
export * from './tempChatRetention';
export * from './retention';
export { tenantSafeBulkWrite } from './tenantBulkWrite';
diff --git a/packages/data-schemas/src/utils/search.spec.ts b/packages/data-schemas/src/utils/search.spec.ts
new file mode 100644
index 0000000000..db38aebcb6
--- /dev/null
+++ b/packages/data-schemas/src/utils/search.spec.ts
@@ -0,0 +1,215 @@
+import {
+ withSearchTokens,
+ computeSearchTokens,
+ normalizeSearchText,
+ buildUserSearchFilter,
+ MAX_SEARCH_TOKENS,
+ USER_SEARCH_TOKEN_FIELDS,
+ MAX_SEARCH_TOKEN_LENGTH,
+ MAX_SEARCH_QUERY_TOKENS,
+} from './search';
+
+describe('normalizeSearchText', () => {
+ it('lowercases and folds accents and compatibility forms', () => {
+ expect(normalizeSearchText('José ÅNGSTRÖM')).toBe('jose angstrom');
+ expect(normalizeSearchText('Full')).toBe('full');
+ expect(normalizeSearchText('İstanbul')).toBe('istanbul');
+ });
+});
+
+describe('computeSearchTokens', () => {
+ it('splits names on whitespace and punctuation', () => {
+ expect(computeSearchTokens('words', " Mary-Jane O'Brien, Jr. ")).toEqual([
+ 'mary',
+ 'jane',
+ 'o',
+ 'brien',
+ 'jr',
+ ]);
+ });
+
+ it('keeps non-Latin scripts as words', () => {
+ expect(computeSearchTokens('words', '山田 太郎')).toEqual(['山田', '太郎']);
+ expect(computeSearchTokens('words', 'Ζωή Κ.')).toEqual(['ζωη', 'κ']);
+ });
+
+ it('indexes the full email, local-part words, and domain labels', () => {
+ expect(computeSearchTokens('email', '[email protected]')).toEqual([
+ 'john',
+ 'smith',
+ 'work',
+ 'mail',
+ 'example',
+ 'co',
+ 'com',
+ 'example-co',
+ ]);
+ });
+
+ it('indexes the full username and its parts', () => {
+ expect(computeSearchTokens('handle', 'jane_doe.99')).toEqual([
+ 'jane_doe.99',
+ 'jane',
+ 'doe',
+ '99',
+ ]);
+ });
+
+ it('dedupes and drops empties', () => {
+ expect(computeSearchTokens('words', 'Ann ann ANN --')).toEqual(['ann']);
+ expect(computeSearchTokens('words', ' ')).toEqual([]);
+ expect(computeSearchTokens('words', '')).toEqual([]);
+ });
+
+ it('returns no tokens for non-strings', () => {
+ expect(computeSearchTokens('words', undefined)).toEqual([]);
+ expect(computeSearchTokens('email', null)).toEqual([]);
+ expect(computeSearchTokens('handle', 42)).toEqual([]);
+ });
+
+ it('caps token length and token count', () => {
+ const long = 'a'.repeat(MAX_SEARCH_TOKEN_LENGTH + 10);
+ expect(computeSearchTokens('words', long)).toEqual(['a'.repeat(MAX_SEARCH_TOKEN_LENGTH)]);
+
+ const many = Array.from({ length: MAX_SEARCH_TOKENS + 10 }, (_, i) => `w${i}`).join(' ');
+ const tokens = computeSearchTokens('words', many);
+ expect(tokens).toHaveLength(MAX_SEARCH_TOKENS);
+ expect(tokens[0]).toBe('w0');
+ });
+
+ it('does not split a surrogate pair when truncating', () => {
+ const [token] = computeSearchTokens('words', '\u{10428}'.repeat(MAX_SEARCH_TOKEN_LENGTH + 5));
+ expect(Array.from(token)).toHaveLength(MAX_SEARCH_TOKEN_LENGTH);
+ expect(token).toBe('\u{10428}'.repeat(MAX_SEARCH_TOKEN_LENGTH));
+ });
+});
+
+describe('withSearchTokens', () => {
+ it('derives tokens for each source field in $set and leaves the input untouched', () => {
+ const update = { $set: { name: 'Ana Lima', role: 'USER' } };
+ const next = withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update);
+ expect(next).toEqual({ $set: { name: 'Ana Lima', role: 'USER', nameTokens: ['ana', 'lima'] } });
+ expect(update).toEqual({ $set: { name: 'Ana Lima', role: 'USER' } });
+ });
+
+ it('handles top-level, $setOnInsert and $unset writes', () => {
+ expect(
+ withSearchTokens(USER_SEARCH_TOKEN_FIELDS, {
+ username: 'al',
+ $setOnInsert: { email: '[email protected]' },
+ $unset: { name: '' },
+ }),
+ ).toEqual({
+ username: 'al',
+ usernameTokens: ['al'],
+ $setOnInsert: { email: '[email protected]', emailTokens: ['[email protected]', 'a', 'b', 'io'] },
+ $unset: { name: '' },
+ $set: { nameTokens: [] },
+ });
+ });
+
+ it('ignores undefined assignments, which Mongoose drops from the update', () => {
+ const update = { $set: { name: undefined, role: 'ADMIN' }, username: undefined };
+ expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update);
+ });
+
+ it('ignores an undefined $unset, which Mongoose also drops', () => {
+ const update = { $unset: { name: undefined } };
+ expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update);
+ });
+
+ it('initializes every unwritten token field on upsert inserts', () => {
+ expect(
+ withSearchTokens(USER_SEARCH_TOKEN_FIELDS, { $set: { name: 'Al' } }, { upsert: true }),
+ ).toEqual({
+ $set: { name: 'Al', nameTokens: ['al'] },
+ $setOnInsert: { emailTokens: [], usernameTokens: [] },
+ });
+ });
+
+ it('takes upsert insert tokens from equality conditions the insert copies', () => {
+ expect(
+ withSearchTokens(
+ USER_SEARCH_TOKEN_FIELDS,
+ { $set: { name: 'Al' } },
+ { upsert: true, filter: { email: '[email protected]', username: { $eq: 'al' } } },
+ ),
+ ).toEqual({
+ $set: { name: 'Al', nameTokens: ['al'] },
+ $setOnInsert: { emailTokens: ['[email protected]', 'al', 'x', 'io'], usernameTokens: ['al'] },
+ });
+ expect(
+ withSearchTokens(
+ USER_SEARCH_TOKEN_FIELDS,
+ { $set: { name: 'Al' } },
+ { upsert: true, filter: { email: { $in: ['[email protected]', '[email protected]'] } } },
+ ),
+ ).toEqual({
+ $set: { name: 'Al', nameTokens: ['al'] },
+ $setOnInsert: { usernameTokens: [] },
+ });
+ });
+
+ it('returns the same reference when no source field is written', () => {
+ const update = { $set: { role: 'ADMIN' } };
+ expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update);
+ const pipeline = [{ $set: { name: 'x' } }];
+ expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, pipeline)).toBe(pipeline);
+ });
+});
+
+describe('buildUserSearchFilter', () => {
+ it('returns null when the query has nothing to match', () => {
+ expect(buildUserSearchFilter('')).toBeNull();
+ expect(buildUserSearchFilter(' ')).toBeNull();
+ });
+
+ it('matches a single token by anchored, case-sensitive prefix on every token field', () => {
+ const filter = buildUserSearchFilter(' José ') as { $or: Array<Record<string, unknown>> };
+ const prefixes = filter.$or.filter((branch) => Object.keys(branch).length === 1);
+ expect(prefixes).toEqual([
+ { nameTokens: /^jose/ },
+ { emailTokens: /^jose/ },
+ { usernameTokens: /^jose/ },
+ ]);
+ expect(filter.$or).toContainEqual({ nameTokens: { $exists: false }, name: /José/i });
+ });
+
+ it('anchors the legacy fallback for callers whose old search was a prefix match', () => {
+ const filter = buildUserSearchFilter('mith', { legacyPrefix: true })!;
+ expect(filter.$or).toContainEqual({ nameTokens: { $exists: false }, name: /^mith/i });
+ expect(filter.$or).not.toContainEqual({ nameTokens: { $exists: false }, name: /mith/i });
+ });
+
+ it('requires every query token and also tries the whole query on full-value fields', () => {
+ const filter = buildUserSearchFilter('john smi') as { $and: Array<{ $or: unknown[] }> };
+ expect(filter.$and).toHaveLength(2);
+ expect(filter.$and[0].$or).toContainEqual({ nameTokens: /^john/ });
+ expect(filter.$and[1].$or).toContainEqual({ nameTokens: /^smi/ });
+ expect(filter.$and[1].$or).toContainEqual({ emailTokens: /^john smi/ });
+ expect(filter.$and[1].$or).not.toContainEqual({ nameTokens: /^john smi/ });
+ });
+
+ it('escapes regex metacharacters and requires every query token up to the cap', () => {
+ const filter = buildUserSearchFilter('a.b') as { $and: Array<{ $or: unknown[] }> };
+ expect(filter.$and[0].$or).toContainEqual({ emailTokens: /^a\.b/ });
+
+ const words = (n: number) => Array.from({ length: n }, (_, i) => `t${i}`).join(' ');
+ const all = buildUserSearchFilter(words(MAX_SEARCH_QUERY_TOKENS)) as {
+ $and: Array<{ $or: unknown[] }>;
+ };
+ expect(all.$and).toHaveLength(MAX_SEARCH_QUERY_TOKENS);
+ expect(all.$and[MAX_SEARCH_QUERY_TOKENS - 1].$or).toContainEqual({
+ nameTokens: new RegExp(`^t${MAX_SEARCH_QUERY_TOKENS - 1}`),
+ });
+ /** Over the cap, nothing matches: extra words are never silently dropped. */
+ expect(buildUserSearchFilter(words(MAX_SEARCH_QUERY_TOKENS + 1))).toBeNull();
+ });
+
+ it('searches punctuation-only queries on full values only', () => {
+ const filter = buildUserSearchFilter('.*') as { $or: unknown[] };
+ expect(filter.$or).toContainEqual({ emailTokens: /^\.\*/ });
+ expect(filter.$or).not.toContainEqual({ nameTokens: expect.anything() });
+ });
+});
diff --git a/packages/data-schemas/src/utils/search.ts b/packages/data-schemas/src/utils/search.ts
new file mode 100644
index 0000000000..63a232561d
--- /dev/null
+++ b/packages/data-schemas/src/utils/search.ts
@@ -0,0 +1,328 @@
+import { escapeRegExp } from './string';
+
+/**
+ * Word-prefix search over derived token arrays.
+ *
+ * Each searchable source field (`name`, `email`, `username`) gets a lowercase,
+ * accent-folded token array stored beside it and indexed with `tenantId`. A query
+ * is tokenized the same way, and each query token must prefix-match some token,
+ * so "john smi" finds "John Smith" through an anchored, case-sensitive regex
+ * that the multikey index can bound. Matches inside a word ("ohn") are not
+ * found, by design: that is the price of an index-bound search.
+ *
+ * The token rules are a stored format: changing them requires re-running the
+ * backfill, so the caps are constants rather than configuration.
+ */
+
+/** How a source value is split into tokens. */
+export type SearchTokenKind = 'words' | 'email' | 'handle';
+
+export interface SearchTokenField {
+ /** The stored field the tokens derive from. */
+ readonly source: string;
+ /** The derived, indexed token array. */
+ readonly tokens: string;
+ readonly kind: SearchTokenKind;
+}
+
+export const USER_SEARCH_TOKEN_FIELDS: readonly SearchTokenField[] = [
+ { source: 'name', tokens: 'nameTokens', kind: 'words' },
+ { source: 'email', tokens: 'emailTokens', kind: 'email' },
+ { source: 'username', tokens: 'usernameTokens', kind: 'handle' },
+];
+
+export const GROUP_SEARCH_TOKEN_FIELDS: readonly SearchTokenField[] = [
+ { source: 'name', tokens: 'nameTokens', kind: 'words' },
+ { source: 'email', tokens: 'emailTokens', kind: 'email' },
+];
+
+/** Longest stored or queried word token; longer words are truncated to this prefix. */
+export const MAX_SEARCH_TOKEN_LENGTH = 64;
+/** Longest whole-value token (a full email address or username). */
+export const MAX_SEARCH_VALUE_LENGTH = 256;
+/** Most tokens stored per field. */
+export const MAX_SEARCH_TOKENS = 32;
+/**
+ * Most distinct query words. Every word must match, so a longer query matches
+ * nothing rather than ignoring its extra words; the cap bounds the filter a
+ * single request can build.
+ */
+export const MAX_SEARCH_QUERY_TOKENS = 16;
+
+const NON_WORD = /[^\p{L}\p{N}]+/u;
+const COMBINING_MARKS = /\p{M}+/gu;
+
+function truncate(value: string, max: number): string {
+ if (value.length <= max) {
+ return value;
+ }
+ return Array.from(value).slice(0, max).join('');
+}
+
+/** Lowercases and folds accents, so "José" and "jose" compare equal. */
+export function normalizeSearchText(value: string): string {
+ return value.normalize('NFKD').replace(COMBINING_MARKS, '').toLowerCase();
+}
+
+function words(normalized: string): string[] {
+ return normalized
+ .split(NON_WORD)
+ .filter(Boolean)
+ .map((word) => truncate(word, MAX_SEARCH_TOKEN_LENGTH));
+}
+
+/**
+ * A value reduced to the words the filter matches, joined by single spaces, so
+ * relevance scoring sees "Mary-Jane" and "mary jane" as the same text.
+ */
+export function normalizeSearchPhrase(value: string): string {
+ return words(normalizeSearchText(value)).join(' ');
+}
+
+/**
+ * Relevance of a matched document for `query`, over the values the filter
+ * searched. Like the filter, it ignores word order: 100 when a value has the
+ * query's words in any order, 80 when a value starts with the query, 50 when a
+ * value contains it or every query word prefixes some word (in one value or
+ * across them), 10 otherwise.
+ */
+export function scoreSearchMatch(
+ values: ReadonlyArray<string | null | undefined>,
+ query: string,
+): number {
+ const phrase = normalizeSearchPhrase(query);
+ if (!phrase) {
+ return 10;
+ }
+ const queryWords = phrase.split(' ');
+ const sortedQuery = [...queryWords].sort().join(' ');
+ const valueWords: string[] = [];
+ let score = 10;
+ for (const value of values) {
+ if (!value) {
+ continue;
+ }
+ const field = normalizeSearchPhrase(value);
+ const words = field.split(' ');
+ if (field === phrase || [...words].sort().join(' ') === sortedQuery) {
+ return 100;
+ }
+ if (field.startsWith(phrase)) {
+ score = Math.max(score, 80);
+ } else if (field.includes(phrase)) {
+ score = Math.max(score, 50);
+ }
+ valueWords.push(...words);
+ }
+ const everyWord = queryWords.every((queryWord) =>
+ valueWords.some((word) => word.startsWith(queryWord)),
+ );
+ return everyWord ? Math.max(score, 50) : score;
+}
+
+/** Derives the stored token array for one field value. Non-strings yield no tokens. */
+export function computeSearchTokens(kind: SearchTokenKind, value: unknown): string[] {
+ if (typeof value !== 'string') {
+ return [];
+ }
+ const normalized = normalizeSearchText(value.trim());
+ if (!normalized) {
+ return [];
+ }
+ const tokens: string[] = [];
+ if (kind !== 'words') {
+ tokens.push(truncate(normalized, MAX_SEARCH_VALUE_LENGTH));
+ }
+ tokens.push(...words(normalized));
+ if (kind === 'email') {
+ const domain = normalized.slice(normalized.lastIndexOf('@') + 1);
+ tokens.push(...domain.split('.').filter(Boolean));
+ }
+ return [...new Set(tokens)].slice(0, MAX_SEARCH_TOKENS);
+}
+
+/** Every token field of `fields`, computed from a document's source values. */
+export function computeSearchTokenSet(
+ fields: readonly SearchTokenField[],
+ doc: Record<string, unknown>,
+): Record<string, string[]> {
+ const result: Record<string, string[]> = {};
+ for (const field of fields) {
+ result[field.tokens] = computeSearchTokens(field.kind, doc[field.source]);
+ }
+ return result;
+}
+
+type UpdateDoc = Record<string, unknown>;
+
+/**
+ * The indexes behind each token field. Tenant first serves tenant-scoped
+ * searches with an equality seek before the prefix range; token first serves
+ * unscoped ones (single-tenant deployments run without a tenant context), which
+ * could not use the tenant-first index at all.
+ */
+export function searchTokenIndexes(field: SearchTokenField): Array<Record<string, 1>> {
+ return [
+ { tenantId: 1, [field.tokens]: 1 },
+ { [field.tokens]: 1, tenantId: 1 },
+ ];
+}
+
+/** Mongoose drops `undefined` assignments from an update, so they write nothing. */
+function writes(doc: UpdateDoc | undefined, key: string): doc is UpdateDoc {
+ return doc != null && key in doc && doc[key] !== undefined;
+}
+
+const SKIP = Symbol('skip');
+
+/**
+ * The value an upsert insert takes for `key` from the query filter: a plain or
+ * `$eq` equality is copied into the new document, no condition leaves it unset.
+ * Any other condition is ambiguous, so the token field is left for the backfill.
+ */
+function insertedValue(filter: UpdateDoc | undefined, key: string): unknown {
+ const condition = filter?.[key];
+ if (condition === undefined) {
+ return undefined;
+ }
+ if (typeof condition === 'string') {
+ return condition;
+ }
+ const eq = operator({ condition }, 'condition');
+ return eq && Object.keys(eq).length === 1 && typeof eq.$eq === 'string' ? eq.$eq : SKIP;
+}
+
+function operator(update: UpdateDoc, key: string): UpdateDoc | undefined {
+ const value = update[key];
+ return value != null && typeof value === 'object' && !Array.isArray(value)
+ ? (value as UpdateDoc)
+ : undefined;
+}
+
+/**
+ * Returns `update` with the token fields recomputed for every source field it
+ * writes, through `$set`, `$setOnInsert`, `$unset` or a top-level replacement
+ * value. Each token field depends on one source field, so a partial update
+ * stays correct. On an upsert, a token field whose source the update does not
+ * write is initialized on insert from the source's equality condition in
+ * `filter` (MongoDB copies it into the new document), or to `[]` when the
+ * filter has none, so new documents never look un-migrated. Copy-on-write: the same reference comes back when nothing
+ * changes. Pipeline updates are returned untouched.
+ *
+ * Mongoose runs no middleware for `bulkWrite`; a bulk write that changes a
+ * source field must pass its update through this function.
+ */
+export function withSearchTokens<T>(
+ fields: readonly SearchTokenField[],
+ update: T,
+ options: { upsert?: boolean; filter?: UpdateDoc } = {},
+): T {
+ if (update == null || typeof update !== 'object' || Array.isArray(update)) {
+ return update;
+ }
+ const source = update as UpdateDoc;
+ let next: UpdateDoc | undefined;
+ const write = (op: string | null, key: string, tokens: string[]) => {
+ next ??= { ...source };
+ if (op == null) {
+ next[key] = tokens;
+ return;
+ }
+ next[op] = { ...(operator(next, op) ?? {}), [key]: tokens };
+ };
+
+ for (const field of fields) {
+ const set = operator(source, '$set');
+ const setOnInsert = operator(source, '$setOnInsert');
+ const unset = operator(source, '$unset');
+ let written = true;
+ if (writes(set, field.source)) {
+ write('$set', field.tokens, computeSearchTokens(field.kind, set[field.source]));
+ } else if (writes(source, field.source)) {
+ write(null, field.tokens, computeSearchTokens(field.kind, source[field.source]));
+ } else if (writes(unset, field.source)) {
+ write('$set', field.tokens, []);
+ } else {
+ written = false;
+ }
+ if (writes(setOnInsert, field.source)) {
+ write(
+ '$setOnInsert',
+ field.tokens,
+ computeSearchTokens(field.kind, setOnInsert[field.source]),
+ );
+ } else if (options.upsert && !written) {
+ const condition = insertedValue(options.filter, field.source);
+ if (condition !== SKIP) {
+ write('$setOnInsert', field.tokens, computeSearchTokens(field.kind, condition));
+ }
+ }
+ }
+ return (next ?? source) as T;
+}
+
+export interface SearchFilterOptions {
+ /** Anchor the legacy fallback at the start of the field, for callers whose old search was a prefix match. */
+ legacyPrefix?: boolean;
+}
+
+/**
+ * Builds the filter for a word-prefix search, or `null` when the query has
+ * nothing to match.
+ *
+ * Every query token must prefix-match a token of some field (fields may
+ * differ per token, so "john gmail" finds John at gmail.com). Fields that
+ * store their whole value (email, username) also match the whole query as a
+ * prefix, so a pasted "john.smith@exa" works. Documents written before the
+ * token fields existed fall back to the caller's old case-insensitive regex
+ * on the source field (unanchored, or anchored with `legacyPrefix`); the
+ * fallback branches seek the token index on `$exists: false` and match
+ * nothing once the backfill has run.
+ */
+export function buildSearchTokenFilter(
+ fields: readonly SearchTokenField[],
+ query: string,
+ options: SearchFilterOptions = {},
+): Record<string, unknown> | null {
+ const trimmed = query.trim();
+ const normalized = normalizeSearchText(trimmed);
+ const queryTokens = [...new Set(words(normalized))];
+ const whole = truncate(normalized, MAX_SEARCH_VALUE_LENGTH);
+ const matchWhole = whole.length > 0 && !(queryTokens.length === 1 && queryTokens[0] === whole);
+ if (
+ !trimmed ||
+ queryTokens.length > MAX_SEARCH_QUERY_TOKENS ||
+ (queryTokens.length === 0 && !matchWhole)
+ ) {
+ return null;
+ }
+
+ const legacy = new RegExp(`${options.legacyPrefix ? '^' : ''}${escapeRegExp(trimmed)}`, 'i');
+ const shared: Record<string, unknown>[] = [];
+ for (const field of fields) {
+ if (matchWhole && field.kind !== 'words') {
+ shared.push({ [field.tokens]: new RegExp(`^${escapeRegExp(whole)}`) });
+ }
+ shared.push({ [field.tokens]: { $exists: false }, [field.source]: legacy });
+ }
+
+ if (queryTokens.length === 0) {
+ return { $or: shared };
+ }
+ const clauses = queryTokens.map((token) => {
+ const prefix = new RegExp(`^${escapeRegExp(token)}`);
+ return { $or: [...fields.map((field) => ({ [field.tokens]: prefix })), ...shared] };
+ });
+ return clauses.length === 1 ? clauses[0] : { $and: clauses };
+}
+
+export function buildUserSearchFilter(
+ query: string,
+ options?: SearchFilterOptions,
+): Record<string, unknown> | null {
+ return buildSearchTokenFilter(USER_SEARCH_TOKEN_FIELDS, query, options);
+}
+
+export function buildGroupSearchFilter(query: string): Record<string, unknown> | null {
+ return buildSearchTokenFilter(GROUP_SEARCH_TOKEN_FIELDS, query);
+}