Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🔎 perf: Index-Bound Word-Prefix Search for Users and Groups (#16719)
Marco Berettade9d46fcd4f4
diff --git a/api/server/services/start/__tests__/migration.spec.js b/api/server/services/start/__tests__/migration.spec.jsnew file mode 100644index 0000000000..ba4bfc32d8--- /dev/null+++ b/api/server/services/start/__tests__/migration.spec.js@@ -0,0 +1,37 @@+const mockWarnOnMissingSearchTokens = jest.fn();+const mockCheckAgentPermissionsMigration = jest.fn();+const mockCheckPromptPermissionsMigration = jest.fn();++jest.mock('@librechat/data-schemas', () => ({+ logger: { error: jest.fn() },+ warnOnMissingSearchTokens: (...args) => mockWarnOnMissingSearchTokens(...args),+}));+jest.mock('@librechat/api', () => ({+ logAgentMigrationWarning: jest.fn(),+ logPromptMigrationWarning: jest.fn(),+ checkAgentPermissionsMigration: (...args) => mockCheckAgentPermissionsMigration(...args),+ checkPromptPermissionsMigration: (...args) => mockCheckPromptPermissionsMigration(...args),+}));+jest.mock('~/models', () => ({ findRoleByIdentifier: jest.fn() }));++const { checkMigrations } = require('../migration');++describe('checkMigrations', () => {+ it('starts the search token probe while the permission checks are still running', async () => {+ let finishAgentCheck;+ mockCheckAgentPermissionsMigration.mockReturnValue(+ new Promise((resolve) => {+ finishAgentCheck = resolve;+ }),+ );+ mockCheckPromptPermissionsMigration.mockResolvedValue({});+ mockWarnOnMissingSearchTokens.mockResolvedValue(undefined);++ const done = checkMigrations();+ expect(mockWarnOnMissingSearchTokens).toHaveBeenCalledTimes(1);++ finishAgentCheck({});+ await done;+ expect(mockCheckPromptPermissionsMigration).toHaveBeenCalledTimes(1);+ });+});diff --git a/api/server/services/start/migration.js b/api/server/services/start/migration.jsindex 70f8300e08..d170590168 100644--- a/api/server/services/start/migration.js+++ b/api/server/services/start/migration.js@@ -1,5 +1,5 @@const mongoose = require('mongoose');-const { logger } = require('@librechat/data-schemas');+const { logger, warnOnMissingSearchTokens } = require('@librechat/data-schemas');const {logAgentMigrationWarning,logPromptMigrationWarning,@@ -13,6 +13,8 @@ const { findRoleByIdentifier } = require('~/models');* This runs at the end to ensure all systems are initialized*/async function checkMigrations() {+ /** Started first so it overlaps the permission checks; it logs its own failures. */+ const searchTokens = warnOnMissingSearchTokens(mongoose.connection);try {const agentMigrationResult = await checkAgentPermissionsMigration({mongoose,@@ -37,6 +39,7 @@ async function checkMigrations() {} catch (error) {logger.error('Failed to check prompt permissions migration:', error);}+ await searchTokens;}module.exports = {diff --git a/config/migrate-search-tokens.js b/config/migrate-search-tokens.jsnew file mode 100644index 0000000000..812e0e0338--- /dev/null+++ b/config/migrate-search-tokens.js@@ -0,0 +1,30 @@+require('dotenv').config();+const mongoose = require('mongoose');+const { backfillSearchTokens } = require('@librechat/data-schemas');+const connect = require('./connect');++/**+ * Backfills the word-prefix search tokens on users and groups saved before they existed.+ * Idempotent; safe to run while the server is up.+ *+ * Usage: npm run migrate:search-tokens [-- --dry-run] [-- --batch-size=500]+ */+(async () => {+ let exitCode = 0;+ try {+ await connect();+ const batchArg = process.argv.find((arg) => arg.startsWith('--batch-size='));+ const result = await backfillSearchTokens(mongoose.connection, {+ dryRun: process.argv.includes('--dry-run'),+ batchSize: batchArg ? parseInt(batchArg.split('=')[1], 10) || undefined : undefined,+ });+ console.log(JSON.stringify(result, null, 2));+ } catch (error) {+ console.error('Search token migration failed:', error);+ exitCode = 1;+ } finally {+ await mongoose.disconnect();+ }+ /** Exit explicitly: with Redis enabled, the cache clients opened on connect keep the process alive. */+ process.exit(exitCode);+})();diff --git a/e2e/specs/mock/db.ts b/e2e/specs/mock/db.tsindex 40127f7135..4a0492eb7a 100644--- a/e2e/specs/mock/db.ts+++ b/e2e/specs/mock/db.ts@@ -16,7 +16,7 @@ function getRuntimeEnvPath(): string {);}-function getMongoUri(): string {+export function getMongoUri(): string {try {const env = JSON.parse(fs.readFileSync(getRuntimeEnvPath(), 'utf8')) as { MONGO_URI?: string };if (env.MONGO_URI) {diff --git a/e2e/specs/mock/scenarios/people-search-tokens.spec.ts b/e2e/specs/mock/scenarios/people-search-tokens.spec.tsnew file mode 100644index 0000000000..5bef523e2b--- /dev/null+++ b/e2e/specs/mock/scenarios/people-search-tokens.spec.ts@@ -0,0 +1,226 @@+import path from 'path';+import { promisify } from 'util';+import { execFile } from 'child_process';+import { expect, test } from '@playwright/test';+import type { Page, Locator } from '@playwright/test';+import { MOCK_ENDPOINTS, NEW_CHAT_PATH, getAccessToken, requestJson, uniqueName } from '../helpers';+import { openAgentBuilder, uniqueAgentName, cleanupAgent } from '../agents.helpers';+import { withMongo, getMongoUri } from '../db';++/**+ * People search (`GET /api/permissions/search-principals`, the share dialog's+ * picker) matches word prefixes over the derived `nameTokens` / `emailTokens` /+ * `usernameTokens` arrays. These scenarios register users through the real+ * auth route, so the schema middleware writes the tokens, and drive the+ * backfill script (`npm run migrate:search-tokens`) against this run's database+ * for documents a writer left without tokens or with stale ones.+ *+ * The primary e2e user is the first one registered in the run, so it is ADMIN+ * and may search principals and share agents without seeding role permissions+ * (see `principal-type-labels.spec.ts`).+ */++const SEARCH_LABEL = 'Search for people or groups by name or email';+const SEARCH_PATH = '/api/permissions/search-principals';+const REPO_ROOT = path.resolve(__dirname, '../../../..');++type SearchResult = { name?: string; email?: string; type: string };++/** Letters only, so a suffix stays one search word. */+function suffix(): string {+ return Array.from({ length: 7 }, () =>+ String.fromCharCode(97 + Math.floor(Math.random() * 26)),+ ).join('');+}++async function register(page: Page, name: string, email: string, username: string) {+ const password = 'Search-tokens-pass-1';+ const response = await page.request.post('/api/auth/register', {+ data: { name, email, username, password, confirm_password: password },+ });+ expect(response.ok(), await response.text()).toBeTruthy();+}++async function search(page: Page, token: string, q: string): Promise<SearchResult[]> {+ const body = await requestJson<{ results: SearchResult[] }>(page, {+ path: `${SEARCH_PATH}?q=${encodeURIComponent(q)}&limit=20`,+ token,+ });+ return body.results;+}++async function deleteUsers(emails: string[]): Promise<void> {+ await withMongo(async (db) => {+ await db.collection('users').deleteMany({ email: { $in: emails } });+ });+}++async function openShareDialog(page: Page): Promise<{ dialog: Locator; agentId: string }> {+ const agentName = uniqueAgentName('E2E People Search');+ const token = await getAccessToken(page);+ const agent = await requestJson<{ id: string }>(page, {+ path: '/api/agents',+ token,+ method: 'POST',+ body: { name: agentName, provider: MOCK_ENDPOINTS[0].label, model: MOCK_ENDPOINTS[0].model },+ });+ const form = await openAgentBuilder(page);+ await form.getByRole('combobox', { name: 'Agent', exact: true }).click();+ await page.getByRole('option', { name: agentName, exact: true }).click();+ await expect(form.getByLabel('Agent name')).toHaveValue(agentName);+ await page.getByRole('button', { name: `Share ${agentName}` }).click();+ const dialog = page.getByRole('dialog', { name: `Share ${agentName}` });+ await expect(dialog).toBeVisible();+ return { dialog, agentId: agent.id };+}++const execFileAsync = promisify(execFile);++/** Async so a stuck script fails the test on its timeout instead of blocking the worker. */+async function runBackfill(): Promise<void> {+ try {+ await execFileAsync('node', ['config/migrate-search-tokens.js'], {+ cwd: REPO_ROOT,+ env: { ...process.env, MONGO_URI: getMongoUri() },+ timeout: 60000,+ killSignal: 'SIGKILL',+ });+ } catch (error) {+ const { stdout = '', stderr = '' } = error as { stdout?: string; stderr?: string };+ throw new Error(+ `Backfill failed: ${String(error)}\n${stdout.slice(-2000)}\n${stderr.slice(-2000)}`,+ );+ }+}++test.describe('people search over word-prefix tokens', () => {+ test('@scenario:people-search-matches-word-prefixes the share dialog finds a user by the start of each name word, and not by a fragment inside a word', async ({+ page,+ }) => {+ test.setTimeout(120000);+ const id = suffix();+ const user = { name: `Quill${id} Brightwater`, email: `${uniqueName('quill')}@search.test` };+ let agentId: string | undefined;+ try {+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });+ await register(page, user.name, user.email, `quill${id}`);++ const opened = await openShareDialog(page);+ agentId = opened.agentId;+ const searchBox = opened.dialog.getByRole('combobox', { name: SEARCH_LABEL });+ await searchBox.fill(`brig quill${id.slice(0, 3)}`);+ await expect(opened.dialog.getByRole('option').filter({ hasText: user.name })).toBeVisible({+ timeout: 10000,+ });++ const token = await getAccessToken(page);+ expect((await search(page, token, `ill${id}`)).map((r) => r.email)).not.toContain(user.email);+ expect((await search(page, token, user.email.slice(0, 12))).map((r) => r.email)).toContain(+ user.email,+ );+ } finally {+ await cleanupAgent(page, agentId);+ await deleteUsers([user.email]);+ }+ });++ test('@scenario:people-search-folds-accents-and-ranks-exact-first an unaccented query finds an accented name and ranks its exact match above a longer prefix match', async ({+ page,+ }) => {+ const id = suffix();+ const exact = { name: `Zoë${id}`, email: `${uniqueName('zoe-exact')}@search.test` };+ const prefix = { name: `Zoe${id}x Park`, email: `${uniqueName('zoe-prefix')}@search.test` };+ try {+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });+ await register(page, prefix.name, prefix.email, `zoep${id}`);+ await register(page, exact.name, exact.email, `zoee${id}`);++ const token = await getAccessToken(page);+ const results = await search(page, token, `zoe${id}`);+ expect(results.map((r) => r.name)).toEqual([exact.name, prefix.name]);+ } finally {+ await deleteUsers([exact.email, prefix.email]);+ }+ });++ test('@scenario:backfill-repairs-missing-and-stale-tokens a user saved without tokens is still found, a user whose tokens went stale is found under the new name once the backfill runs', async ({+ page,+ }) => {+ test.setTimeout(120000);+ const id = suffix();+ const legacy = { name: `Legacy${id} Person`, email: `${uniqueName('legacy')}@search.test` };+ const renamed = { name: `Renamed${id} Person`, email: `${uniqueName('renamed')}@search.test` };+ try {+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });+ const primary = await withMongo((db) =>+ db.collection('users').findOne({}, { sort: { createdAt: 1 } }),+ );+ /** Raw writes, as an older server would leave them: one user with no+ * tokens at all, one renamed without its tokens being rewritten. */+ await withMongo(async (db) => {+ const now = new Date();+ await db.collection('users').insertMany([+ { ...legacy, tenantId: primary?.tenantId, createdAt: now, updatedAt: now },+ {+ ...renamed,+ tenantId: primary?.tenantId,+ nameTokens: [`original${id}`, 'person'],+ emailTokens: [renamed.email],+ usernameTokens: [],+ createdAt: now,+ updatedAt: now,+ },+ ]);+ });++ const token = await getAccessToken(page);+ const emails = async (q: string) => (await search(page, token, q)).map((r) => r.email);+ expect(await emails(`gacy${id}`)).toContain(legacy.email);+ expect(await emails(`renamed${id}`)).not.toContain(renamed.email);++ await runBackfill();++ expect(await emails(`renamed${id}`)).toContain(renamed.email);+ expect(await emails(`original${id}`)).not.toContain(renamed.email);+ expect(await emails(`legacy${id}`)).toContain(legacy.email);+ const stored = await withMongo((db) =>+ db.collection('users').findOne({ email: legacy.email }),+ );+ expect(stored?.nameTokens).toEqual([`legacy${id}`, 'person']);+ } finally {+ await deleteUsers([legacy.email, renamed.email]);+ }+ });++ test('@scenario:people-picker-searches-once-per-pause typing a query character by character sends one search request, not one per keystroke', async ({+ page,+ }) => {+ test.setTimeout(120000);+ const id = suffix();+ const user = { name: `Pause${id} Typist`, email: `${uniqueName('pause')}@search.test` };+ let agentId: string | undefined;+ try {+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });+ await register(page, user.name, user.email, `pause${id}`);+ const opened = await openShareDialog(page);+ agentId = opened.agentId;++ const queries: string[] = [];+ page.on('request', (request) => {+ const url = new URL(request.url());+ if (url.pathname === SEARCH_PATH) {+ queries.push(url.searchParams.get('q') ?? '');+ }+ });+ const searchBox = opened.dialog.getByRole('combobox', { name: SEARCH_LABEL });+ await searchBox.pressSequentially(`pause${id}`, { delay: 40 });+ await expect(opened.dialog.getByRole('option').filter({ hasText: user.name })).toBeVisible({+ timeout: 10000,+ });+ expect(queries).toEqual([`pause${id}`]);+ } finally {+ await cleanupAgent(page, agentId);+ await deleteUsers([user.email]);+ }+ });+});diff --git a/package.json b/package.jsonindex 468915b789..72dde5ff7f 100644--- a/package.json+++ b/package.json@@ -143,6 +143,8 @@"migrate:tenant-indexes": "node config/migrate-tenant-indexes.js","migrate:tenant-indexes:dry-run": "node config/migrate-tenant-indexes.js --dry-run","migrate:terms-timestamp": "node config/migrate-terms-timestamp.js",+ "migrate:search-tokens": "node config/migrate-search-tokens.js",+ "migrate:search-tokens:dry-run": "node config/migrate-search-tokens.js --dry-run","flush-cache": "node config/flush-cache.js","migrate:agent-permissions:dry-run": "node config/migrate-agent-permissions.js --dry-run","migrate:agent-permissions": "node config/migrate-agent-permissions.js",diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.tsindex 3c1aea1303..b3582c33f1 100644--- a/packages/api/src/admin/users.spec.ts+++ b/packages/api/src/admin/users.spec.ts@@ -1,4 +1,5 @@import { Types } from 'mongoose';+import { buildUserSearchFilter } from '@librechat/data-schemas';import { PrincipalType, SystemRoles } from 'librechat-data-provider';import type { IUser, UserDeleteResult } from '@librechat/data-schemas';import type { Response } from 'express';@@ -205,20 +206,23 @@ describe('createAdminUsersHandlers', () => {expect(response.capped).toBe(true);});- it('searches name, email, and username with anchored prefix regex', async () => {+ it('searches the indexed word-prefix tokens of name, email, and username', async () => {const findUsers = jest.fn().mockResolvedValue([]);const deps = createDeps({ findUsers });const handlers = createAdminUsersHandlers(deps);- const { req, res } = createReqRes({ query: { q: 'test' } });+ const { req, res } = createReqRes({ query: { q: 'Test' } });await handlers.searchUsers(req, res);const filter = findUsers.mock.calls[0][0];- expect(filter.$or).toHaveLength(3);- expect(filter.$or[0]).toHaveProperty('name');- expect(filter.$or[1]).toHaveProperty('email');- expect(filter.$or[2]).toHaveProperty('username');- expect(filter.$or[0].name.source).toBe('^test');+ expect(filter).toEqual(buildUserSearchFilter('Test', { legacyPrefix: true }));+ expect(filter.$or).toEqual(+ expect.arrayContaining([+ { nameTokens: /^test/ },+ { emailTokens: /^test/ },+ { usernameTokens: /^test/ },+ ]),+ );});it('projects username in the field selection', async () => {@@ -242,8 +246,25 @@ describe('createAdminUsersHandlers', () => {await handlers.searchUsers(req, res);const filter = findUsers.mock.calls[0][0];- expect(filter.$or[0].name).toBeInstanceOf(RegExp);- expect(filter.$or[0].name.source).toBe('^test\\.user\\+1');+ expect(filter.$and[0].$or).toContainEqual({ emailTokens: /^test\.user\+1/ });+ /** Documents awaiting the backfill keep the old anchored prefix match. */+ expect(filter.$and[0].$or).toContainEqual({+ nameTokens: { $exists: false },+ name: /^test\.user\+1/i,+ });+ });++ it('returns no users without querying when the query has nothing to match', async () => {+ const findUsers = jest.fn();+ const deps = createDeps({ findUsers });+ const handlers = createAdminUsersHandlers(deps);+ const { req, res, status, json } = createReqRes({ query: { q: '\u0301\u0301' } });++ await handlers.searchUsers(req, res);++ expect(findUsers).not.toHaveBeenCalled();+ expect(status).toHaveBeenCalledWith(200);+ expect(json).toHaveBeenCalledWith({ users: [], total: 0, capped: false });});it('returns 400 when query is missing', async () => {diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.tsindex ae8ac19c9c..d28569e50a 100644--- a/packages/api/src/admin/users.ts+++ b/packages/api/src/admin/users.ts@@ -1,7 +1,7 @@import { Types } from 'mongoose';import { randomUUID } from 'node:crypto';import { PrincipalType, SystemRoles } from 'librechat-data-provider';-import { logger, isValidObjectIdString } from '@librechat/data-schemas';+import { logger, isValidObjectIdString, buildUserSearchFilter } from '@librechat/data-schemas';import type {IUser,IConfig,@@ -197,14 +197,13 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {}const searchLimit = Math.min(Math.max(1, parseInt(limitStr, 10) || 20), 50);- const escaped = trimmed.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');- const regex = new RegExp(`^${escaped}`, 'i');-- const users = await findUsers(- { $or: [{ name: regex }, { email: regex }, { username: regex }] },- '_id name email username avatar',- { limit: searchLimit, sort: { name: 1 } },- );+ const filter = buildUserSearchFilter(trimmed, { legacyPrefix: true });+ const users = filter+ ? await findUsers(filter, '_id name email username avatar', {+ limit: searchLimit,+ sort: { name: 1 },+ })+ : [];const results: AdminUserSearchResult[] = users.map((u) => ({id: u._id?.toString() ?? '',diff --git a/packages/data-schemas/src/index.ts b/packages/data-schemas/src/index.tsindex 560d21ebdb..38b269be70 100644--- a/packages/data-schemas/src/index.ts+++ b/packages/data-schemas/src/index.ts@@ -117,4 +117,7 @@ export {migrateTenantIndexes,dropSupersededPromptGroupIndexes,backfillMCPServerNormalizedNames,+ backfillSearchTokens,+ warnOnMissingSearchTokens,} from './migrations';+export type { SearchTokenBackfillResult } from './migrations';diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.tsindex d9318173d1..7bffe5e381 100644--- a/packages/data-schemas/src/methods/user.methods.spec.ts+++ b/packages/data-schemas/src/methods/user.methods.spec.ts@@ -1842,6 +1842,35 @@ describe('User Methods - Database Tests', () => {expect(results.length).toBeGreaterThan(0);});+ test('should rank an accent-folded exact match above a prefix match', async () => {+ await User.create([+ { name: 'Zoe Adams', email: '[email protected]', username: 'zadams', provider: 'local' },+ { name: 'Zoë', email: '[email protected]', username: 'zumlaut', provider: 'local' },+ ]);++ const results = await methods.searchUsers({ searchPattern: 'zoe', limit: 1 });++ expect(results).toHaveLength(1);+ expect((results[0] as unknown as t.IUser).name).toBe('Zoë');+ });++ test('should rank a reordered full-name match above a longer prefix match', async () => {+ await User.create([+ {+ name: 'Smith Johnson',+ email: '[email protected]',+ username: 'sjohnson',+ provider: 'local',+ },+ { name: 'John Smith', email: '[email protected]', username: 'jsmith', provider: 'local' },+ ]);++ const results = await methods.searchUsers({ searchPattern: 'smith john', limit: 1 });++ expect(results).toHaveLength(1);+ expect((results[0] as unknown as t.IUser).name).toBe('John Smith');+ });+test('should respect limit', async () => {const results = await methods.searchUsers({ searchPattern: 'example', limit: 2 });@@ -1869,9 +1898,11 @@ describe('User Methods - Database Tests', () => {});const results = await methods.searchUsers({ searchPattern: '.*' });+ expect(results).toEqual([]);- expect(results).toHaveLength(1);- expect((results[0] as unknown as t.IUser).name).toBe('Literal .* User');+ const literal = await methods.searchUsers({ searchPattern: 'literal' });+ expect(literal).toHaveLength(1);+ expect((literal[0] as unknown as t.IUser).name).toBe('Literal .* User');});test('should handle invalid regex syntax as literal search text', async () => {diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.tsindex cc95028344..94aaddb6f1 100644--- a/packages/data-schemas/src/methods/user.ts+++ b/packages/data-schemas/src/methods/user.ts@@ -16,8 +16,8 @@ import type {} from '~/types';import type { TwoFactorEnrollmentGuard, TwoFactorEnrollmentUpdate } from '~/types';import type { CacheStore } from '~/types';+import { scoreSearchMatch, buildUserSearchFilter } from '~/utils/search';import { evictAuthUserDocs } from '~/utils/eviction';-import { escapeRegExp } from '~/utils/string';import { signPayload } from '~/crypto';import logger from '~/config/winston';@@ -891,7 +891,7 @@ export function createUserMethods(}/**- * Search for users by pattern matching on name, email, or username (case-insensitive)+ * Search for users by word prefix of name, email, or username (see `buildUserSearchFilter`)* @param searchPattern - The pattern to search for* @param limit - Maximum number of results to return* @param fieldsToSelect - The fields to include or exclude in the returned documents@@ -975,12 +975,13 @@ export function createUserMethods(}const trimmedPattern = searchPattern.trim();- const regex = new RegExp(escapeRegExp(trimmedPattern), 'i');+ const filter = buildUserSearchFilter(trimmedPattern);+ if (!filter) {+ return [];+ }const User = mongoose.models.User;- const query = User.find({- $or: [{ email: regex }, { name: regex }, { username: regex }],- }).limit(limit * 2); // Get more results to allow for relevance sorting+ const query = User.find(filter).limit(limit * 2); // Get more results to allow for relevance sortingif (fieldsToSelect) {query.select(fieldsToSelect);@@ -988,41 +989,10 @@ export function createUserMethods(const users = await query.lean<IUser[]>();- // Score results by relevance- const startsWithPattern = trimmedPattern.toLowerCase();-- const scoredUsers = users.map((user) => {- const searchableFields = [user.name, user.email, user.username].filter(- (field): field is string => typeof field === 'string' && field.length > 0,- );- let maxScore = 0;-- for (const field of searchableFields) {- const fieldLower = field.toLowerCase();- let score = 0;-- // Exact match gets highest score- if (fieldLower === startsWithPattern) {- score = 100;- }- // Starts with query gets high score- else if (fieldLower.startsWith(startsWithPattern)) {- score = 80;- }- // Contains query gets medium score- else if (fieldLower.includes(startsWithPattern)) {- score = 50;- }- // Default score for database match- else {- score = 10;- }-- maxScore = Math.max(maxScore, score);- }-- return { ...user, _searchScore: maxScore };- });+ const scoredUsers = users.map((user) => ({+ ...user,+ _searchScore: scoreSearchMatch([user.name, user.email, user.username], trimmedPattern),+ }));/** Top results sorted by relevance */return scoredUsersdiff --git a/packages/data-schemas/src/methods/userGroup.spec.ts b/packages/data-schemas/src/methods/userGroup.spec.tsindex db3212e596..9c4a3a8ce0 100644--- a/packages/data-schemas/src/methods/userGroup.spec.ts+++ b/packages/data-schemas/src/methods/userGroup.spec.ts@@ -117,16 +117,18 @@ describe('userGroup methods', () => {expect(results[0].name).toBe('Design');});- it('matches on description field', async () => {+ it('does not match on the description field (search covers name and email)', async () => {const results = await methods.findGroupsByNamePattern('Eng team');- expect(results).toHaveLength(1);- expect(results[0].name).toBe('Engineering');+ expect(results).toEqual([]);});it('treats regex metacharacters as literal text', async () => {const results = await methods.findGroupsByNamePattern('.*');- expect(results).toHaveLength(1);- expect(results[0].name).toBe('Literal .* Group');+ expect(results).toEqual([]);++ const literal = await methods.findGroupsByNamePattern('literal');+ expect(literal).toHaveLength(1);+ expect(literal[0].name).toBe('Literal .* Group');});it('filters by source when provided', async () => {@@ -1245,6 +1247,14 @@ describe('userGroup methods', () => {expect(score).toBe(50);});+ it('scores accent-folded matches like their plain spelling', () => {+ const score = methods.calculateRelevanceScore(+ { type: PrincipalType.USER, name: 'Zoë', source: 'local' },+ 'zoe',+ );+ expect(score).toBe(100);+ });+it('returns 10 (default) when no substring or exact match', () => {const score = methods.calculateRelevanceScore({ type: PrincipalType.USER, name: 'bob', source: 'local' },@@ -1267,17 +1277,51 @@ describe('userGroup methods', () => {expect(score).toBe(100);});- it('checks description for GROUP type', () => {- const score = methods.calculateRelevanceScore(+ it('ignores description for GROUP type, which search no longer matches', () => {+ const weak = methods.calculateRelevanceScore({type: PrincipalType.GROUP,- name: 'other',- description: 'alice team',+ name: 'Corporate Platform',+ description: 'plat',source: 'local',},- 'alice',+ 'plat',);- expect(score).toBe(80);+ const strong = methods.calculateRelevanceScore(+ { type: PrincipalType.GROUP, name: 'Platform Team', source: 'local' },+ 'plat',+ );+ expect(weak).toBe(50);+ expect(strong).toBeGreaterThan(weak);+ });++ it('scores a name equal to the query up to separators as an exact match', () => {+ const score = methods.calculateRelevanceScore(+ { type: PrincipalType.USER, name: 'Mary-Jane', source: 'local' },+ 'mary jane',+ );+ expect(score).toBe(100);+ });++ it('scores a reordered full-name query as exact, above a longer prefix match', () => {+ const exact = methods.calculateRelevanceScore(+ { type: PrincipalType.USER, name: 'John Smith', source: 'local' },+ 'smith john',+ );+ const prefix = methods.calculateRelevanceScore(+ { type: PrincipalType.USER, name: 'Smith Johnson', source: 'local' },+ 'smith john',+ );+ expect(exact).toBe(100);+ expect(prefix).toBe(80);+ });++ it('scores query words matched across fields like a contains match', () => {+ const score = methods.calculateRelevanceScore(+ { type: PrincipalType.USER, name: 'John Doe', email: '[email protected]', source: 'local' },+ 'john gmail',+ );+ expect(score).toBe(50);});it('picks the highest score across multiple fields', () => {@@ -1402,9 +1446,11 @@ describe('userGroup methods', () => {});const results = await methods.searchPrincipals('.*', 10, [PrincipalType.USER]);+ expect(results).toEqual([]);- expect(results).toHaveLength(1);- expect(results[0].name).toBe('Literal .* User');+ const literal = await methods.searchPrincipals('literal', 10, [PrincipalType.USER]);+ expect(literal).toHaveLength(1);+ expect(literal[0].name).toBe('Literal .* User');});it('handles invalid regex syntax as literal search text', async () => {diff --git a/packages/data-schemas/src/methods/userGroup.ts b/packages/data-schemas/src/methods/userGroup.tsindex 451f8e97c4..feefa83460 100644--- a/packages/data-schemas/src/methods/userGroup.ts+++ b/packages/data-schemas/src/methods/userGroup.ts@@ -4,6 +4,7 @@ import { CacheKeys, PrincipalType, SystemRoles } from 'librechat-data-provider';import type { TPrincipalSearchResult } from 'librechat-data-provider';import type { Model, ClientSession, FilterQuery } from 'mongoose';import type { CacheStore, IGroup, IRole, IUser } from '~/types';+import { buildUserSearchFilter, scoreSearchMatch, buildGroupSearchFilter } from '~/utils/search';import { isValidObjectIdString } from '~/utils/objectId';import { scopedCacheKey } from '~/config/tenantContext';import { escapeRegExp } from '~/utils/string';@@ -607,7 +608,7 @@ export function createUserGroupMethods(}/**- * Find groups by name pattern (case-insensitive partial match)+ * Find groups by word prefix of name or email (see `buildGroupSearchFilter`)* @param namePattern - The name pattern to search for* @param source - Optional source filter ('entra', 'local', or null for all)* @param limit - Maximum number of results to return@@ -621,10 +622,11 @@ export function createUserGroupMethods(session?: ClientSession,): Promise<IGroup[]> {const Group = mongoose.models.Group as Model<IGroup>;- const regex = new RegExp(escapeRegExp(namePattern), 'i');- const query: Record<string, unknown> = {- $or: [{ name: regex }, { email: regex }, { description: regex }],- };+ const filter = buildGroupSearchFilter(namePattern);+ if (!filter) {+ return [];+ }+ const query: Record<string, unknown> = { ...filter };if (source) {query.source = source;@@ -869,39 +871,12 @@ export function createUserGroupMethods(* @returns Relevance score (0-100)*/function calculateRelevanceScore(item: TPrincipalSearchResult, searchPattern: string): number {- const normalizedPattern = searchPattern.toLowerCase();-- /** Get searchable text based on type */- const searchableFields =+ /** The fields each search matches on; a group's description is not one of them */+ const values =item.type === PrincipalType.USER- ? [item.name, item.email, item.username].filter(Boolean)- : [item.name, item.email, item.description].filter(Boolean);-- let maxScore = 0;-- for (const field of searchableFields) {- if (!field) continue;- const fieldLower = field.toLowerCase();- let score = 0;-- /** Exact match gets highest score */- if (fieldLower === normalizedPattern) {- score = 100;- } else if (fieldLower.startsWith(normalizedPattern)) {- /** Starts with query gets high score */- score = 80;- } else if (fieldLower.includes(normalizedPattern)) {- /** Contains query gets medium score */- score = 50;- } else {- /** Default score for database match */- score = 10;- }-- maxScore = Math.max(maxScore, score);- }-- return maxScore;+ ? [item.name, item.email, item.username]+ : [item.name, item.email];+ return scoreSearchMatch(values, searchPattern);}/**@@ -970,7 +945,8 @@ export function createUserGroupMethods(}/**- * Search for principals (users and groups) by pattern matching on name/email+ * Search for principals: users and groups by word prefix (see `buildSearchTokenFilter`),+ * roles by name* Returns combined results in TPrincipalSearchResult format without sorting* @param searchPattern - The pattern to search for* @param limitPerType - Maximum number of results to return@@ -992,17 +968,11 @@ export function createUserGroupMethods(const escapedPattern = escapeRegExp(trimmedPattern);const promises: Promise<TPrincipalSearchResult[]>[] = [];- if (!typeFilter || typeFilter.includes(PrincipalType.USER)) {- /** Note: searchUsers is imported from ~/models and needs to be passed in or implemented */+ const userFilter = buildUserSearchFilter(trimmedPattern);+ if (userFilter && (!typeFilter || typeFilter.includes(PrincipalType.USER))) {const userFields = 'name email username avatar provider idOnTheSource role';- /** For now, we'll use a direct query instead of searchUsers */const User = mongoose.models.User as Model<IUser>;- const regex = new RegExp(escapedPattern, 'i');- const userQuery = User.find({- $or: [{ name: regex }, { email: regex }, { username: regex }],- })- .select(userFields)- .limit(limitPerType);+ const userQuery = User.find(userFilter).select(userFields).limit(limitPerType);if (session) {userQuery.session(session);diff --git a/packages/data-schemas/src/migrations/index.ts b/packages/data-schemas/src/migrations/index.tsindex c3ba619181..ed2699d8a4 100644--- a/packages/data-schemas/src/migrations/index.ts+++ b/packages/data-schemas/src/migrations/index.ts@@ -2,3 +2,5 @@ export { dropSupersededTenantIndexes, migrateTenantIndexes } from './tenantIndexexport { dropSupersededPromptGroupIndexes } from './promptGroupIndexes';export { createMCPAuthorityLookupIndexes } from './mcpAuthorityIndexes';export { MCPServerNameMigrationError, backfillMCPServerNormalizedNames } from './mcpServerNames';+export { backfillSearchTokens, warnOnMissingSearchTokens } from './searchTokens';+export type { SearchTokenBackfillResult } from './searchTokens';diff --git a/packages/data-schemas/src/migrations/searchTokens.spec.ts b/packages/data-schemas/src/migrations/searchTokens.spec.tsnew file mode 100644index 0000000000..38c62f81f5--- /dev/null+++ b/packages/data-schemas/src/migrations/searchTokens.spec.ts@@ -0,0 +1,247 @@+import mongoose from 'mongoose';+import { Collection } from 'mongodb';+import { MongoMemoryServer } from 'mongodb-memory-server';+import type { Document, FindOptions } from 'mongodb';+import {+ searchTokenIndexes,+ USER_SEARCH_TOKEN_FIELDS,+ GROUP_SEARCH_TOKEN_FIELDS,+} from '~/utils/search';+import { backfillSearchTokens, warnOnMissingSearchTokens } from './searchTokens';+import logger from '~/config/winston';++jest.mock('~/config/winston', () => ({+ error: jest.fn(),+ warn: jest.fn(),+ info: jest.fn(),+ debug: jest.fn(),+}));++let mongoServer: MongoMemoryServer;++beforeAll(async () => {+ mongoServer = await MongoMemoryServer.create();+ await mongoose.connect(mongoServer.getUri());+});++afterAll(async () => {+ await mongoose.disconnect();+ await mongoServer.stop();+});++beforeEach(async () => {+ await mongoose.connection.dropDatabase();+ jest.clearAllMocks();+});++const users = () => mongoose.connection.db!.collection('users');+const groups = () => mongoose.connection.db!.collection('groups');++describe('backfillSearchTokens', () => {+ it('backfills users and groups across tenants in batches and is idempotent', async () => {+ await users().insertMany([+ { name: 'Ana Lima', email: '[email protected]', username: 'ana', tenantId: 'tenant-a' },+ { name: 'Ben Ode', email: '[email protected]', tenantId: 'tenant-b' },+ { email: '[email protected]' },+ {+ name: 'Already Done',+ email: '[email protected]',+ nameTokens: ['already', 'done'],+ emailTokens: ['[email protected]', 'done', 'z', 'io'],+ usernameTokens: [],+ },+ ]);+ await groups().insertMany([{ name: 'Core Team', source: 'local' }]);++ const result = await backfillSearchTokens(mongoose.connection, { batchSize: 2 });+ expect(result).toEqual({+ pending: { users: 3, groups: 1 },+ updated: { users: 3, groups: 1 },+ });++ const ana = await users().findOne({ email: '[email protected]' });+ expect(ana).toMatchObject({+ tenantId: 'tenant-a',+ nameTokens: ['ana', 'lima'],+ emailTokens: ['[email protected]', 'ana', 'x', 'io'],+ usernameTokens: ['ana'],+ });+ expect(await users().findOne({ email: '[email protected]' })).toMatchObject({+ nameTokens: [],+ usernameTokens: [],+ });+ expect(await groups().findOne({ name: 'Core Team' })).toMatchObject({+ nameTokens: ['core', 'team'],+ emailTokens: [],+ });++ await expect(backfillSearchTokens(mongoose.connection)).resolves.toEqual({+ pending: { users: 0, groups: 0 },+ updated: { users: 0, groups: 0 },+ });+ });++ it('completes documents that a partial update tokenized for only one field', async () => {+ await users().insertOne({ name: 'Part Ial', email: '[email protected]', emailTokens: ['[email protected]'] });+ await backfillSearchTokens(mongoose.connection);+ expect(await users().findOne({ email: '[email protected]' })).toMatchObject({+ nameTokens: ['part', 'ial'],+ emailTokens: ['[email protected]', 'p', 'x', 'io'],+ });+ });++ it('repairs tokens left stale by a writer that did not maintain them', async () => {+ await users().insertOne({+ name: 'New Name',+ email: '[email protected]',+ nameTokens: ['old', 'name'],+ emailTokens: ['[email protected]', 'stale', 'x', 'io'],+ usernameTokens: [],+ });+ await expect(backfillSearchTokens(mongoose.connection, { dryRun: true })).resolves.toEqual({+ pending: { users: 1, groups: 0 },+ updated: { users: 0, groups: 0 },+ });+ await expect(backfillSearchTokens(mongoose.connection)).resolves.toEqual({+ pending: { users: 1, groups: 0 },+ updated: { users: 1, groups: 0 },+ });+ expect(await users().findOne({ email: '[email protected]' })).toMatchObject({+ nameTokens: ['new', 'name'],+ });+ });++ it('creates the token indexes when the schema indexes were never built', async () => {+ await users().insertOne({ name: 'No Index', email: '[email protected]' });+ await groups().insertOne({ name: 'Plain Group' });+ await backfillSearchTokens(mongoose.connection);+ const userIndexes = (await users().indexes()).map((index) => index.name);+ const groupIndexes = (await groups().indexes()).map((index) => index.name);+ expect(userIndexes).toEqual(+ expect.arrayContaining([+ 'nameTokens_1_tenantId_1',+ 'emailTokens_1_tenantId_1',+ 'usernameTokens_1_tenantId_1',+ 'tenantId_1_nameTokens_1',+ 'tenantId_1_emailTokens_1',+ 'tenantId_1_usernameTokens_1',+ ]),+ );+ expect(groupIndexes).toEqual(+ expect.arrayContaining([+ 'nameTokens_1_tenantId_1',+ 'emailTokens_1_tenantId_1',+ 'tenantId_1_nameTokens_1',+ 'tenantId_1_emailTokens_1',+ ]),+ );+ });++ it('writes nothing on a dry run', async () => {+ await users().insertOne({ name: 'Dry Run', email: '[email protected]' });+ await expect(backfillSearchTokens(mongoose.connection, { dryRun: true })).resolves.toEqual({+ pending: { users: 1, groups: 0 },+ updated: { users: 0, groups: 0 },+ });+ expect(await users().findOne({ email: '[email protected]' })).not.toHaveProperty('nameTokens');+ });+});++describe('warnOnMissingSearchTokens', () => {+ it('warns only while documents lack tokens', async () => {+ await users().insertOne({ name: 'Old User', email: '[email protected]' });+ await groups().insertOne({ name: 'Old Group' });+ await warnOnMissingSearchTokens(mongoose.connection);+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users and groups'));++ jest.clearAllMocks();+ await backfillSearchTokens(mongoose.connection);+ await warnOnMissingSearchTokens(mongoose.connection);+ expect(logger.warn).not.toHaveBeenCalled();+ });++ /** Runs the startup check and explains the `users` probe it actually issued. */+ const explainUserProbe = async () => {+ const findOne = jest.spyOn(Collection.prototype, 'findOne');+ try {+ await warnOnMissingSearchTokens(mongoose.connection);+ const call = findOne.mock.contexts.findIndex(+ (collection) => collection.collectionName === 'users',+ );+ const [filter, options] = findOne.mock.calls[call] as [Document, FindOptions];+ expect(options.maxTimeMS).toBeGreaterThan(0);+ return await users().find(filter).limit(1).explain('executionStats');+ } finally {+ findOne.mockRestore();+ }+ };++ it('warns without probing documents while the token indexes do not exist', async () => {+ await users().insertMany(+ Array.from({ length: 200 }, (_, i) => ({ name: `Old ${i}`, email: `old${i}@x.io` })),+ );+ const findOne = jest.spyOn(Collection.prototype, 'findOne');+ try {+ await warnOnMissingSearchTokens(mongoose.connection);+ const probed = findOne.mock.contexts.some(+ (collection) => collection.collectionName === 'users',+ );+ expect(probed).toBe(false);+ } finally {+ findOne.mockRestore();+ }+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users'));+ });++ it('examines no documents once the backfill has built the indexes and filled every token', async () => {+ await users().insertMany(+ Array.from({ length: 50 }, (_, i) => ({ name: `User ${i}`, email: `u${i}@x.io` })),+ );+ await backfillSearchTokens(mongoose.connection);+ jest.clearAllMocks();+ const explain = await explainUserProbe();+ expect(JSON.stringify(explain.queryPlanner.winningPlan)).not.toContain('COLLSCAN');+ expect(explain.executionStats.totalDocsExamined).toBe(0);+ expect(logger.warn).not.toHaveBeenCalled();+ });++ it('warns when every document has tokens but the token indexes were never built', async () => {+ await users().insertOne({+ name: 'Fresh User',+ email: '[email protected]',+ nameTokens: ['fresh', 'user'],+ emailTokens: ['[email protected]', 'fresh', 'x', 'io'],+ usernameTokens: [],+ });+ await warnOnMissingSearchTokens(mongoose.connection);+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Some users'));++ jest.clearAllMocks();+ await backfillSearchTokens(mongoose.connection);+ await warnOnMissingSearchTokens(mongoose.connection);+ expect(logger.warn).not.toHaveBeenCalled();+ });++ it('stays quiet on a database with no users or groups yet', async () => {+ await warnOnMissingSearchTokens(mongoose.connection);+ expect(logger.warn).not.toHaveBeenCalled();+ expect(logger.error).not.toHaveBeenCalled();+ });++ it('logs a failed probe instead of failing startup', async () => {+ /** Every token index exists, so the check reaches the probe, which then fails. */+ const indexes = [...USER_SEARCH_TOKEN_FIELDS, ...GROUP_SEARCH_TOKEN_FIELDS]+ .flatMap(searchTokenIndexes)+ .map((key) => ({ key }));+ const findOne = jest.fn(() => Promise.reject(new Error('down')));+ const broken = {+ db: { collection: () => ({ indexes: async () => indexes, findOne }) },+ };+ await expect(+ warnOnMissingSearchTokens(broken as unknown as typeof mongoose.connection),+ ).resolves.toBeUndefined();+ expect(findOne).toHaveBeenCalled();+ expect(logger.error).toHaveBeenCalled();+ expect(logger.warn).not.toHaveBeenCalled();+ });+});diff --git a/packages/data-schemas/src/migrations/searchTokens.ts b/packages/data-schemas/src/migrations/searchTokens.tsnew file mode 100644index 0000000000..42d668883a--- /dev/null+++ b/packages/data-schemas/src/migrations/searchTokens.ts@@ -0,0 +1,178 @@+import type { AnyBulkWriteOperation, Document } from 'mongodb';+import type { Connection } from 'mongoose';+import type { SearchTokenField } from '~/utils/search';+import {+ searchTokenIndexes,+ computeSearchTokenSet,+ USER_SEARCH_TOKEN_FIELDS,+ GROUP_SEARCH_TOKEN_FIELDS,+} from '~/utils/search';+import { buildIndexWithRetry } from '~/utils/retry';+import logger from '~/config/winston';++const DEFAULT_BATCH_SIZE = 500;+/** Upper bound on the startup probe, in case it runs before the token indexes exist. */+const STARTUP_PROBE_MAX_TIME_MS = 5_000;++const SEARCH_TOKEN_COLLECTIONS: ReadonlyArray<{+ name: string;+ fields: readonly SearchTokenField[];+}> = [+ { name: 'users', fields: USER_SEARCH_TOKEN_FIELDS },+ { name: 'groups', fields: GROUP_SEARCH_TOKEN_FIELDS },+];++export interface SearchTokenBackfillResult {+ /** Documents whose token fields are missing or no longer match their source, per collection. */+ pending: Record<string, number>;+ /** Documents written, per collection; zero on a dry run. */+ updated: Record<string, number>;+}++/** Index-bound: each branch seeks one token index on its missing-key bounds. */+function missingTokens(fields: readonly SearchTokenField[]): Document {+ return { $or: fields.map((field) => ({ [field.tokens]: { $exists: false } })) };+}++function sameTokens(stored: unknown, expected: string[]): boolean {+ return (+ Array.isArray(stored) &&+ stored.length === expected.length &&+ stored.every((token, index) => token === expected[index])+ );+}++/**+ * Brings the search-token arrays on users and groups in line with their source+ * fields: fills documents saved before the tokens existed and repairs tokens+ * left stale by a writer that did not maintain them (an older server during a+ * rolling deploy, a raw collection write). Every document is read and only the+ * ones whose tokens differ are written, so it is idempotent and resumable; run+ * it again once a rollout completes. Each write is guarded by the source values+ * it was computed from, so a concurrent rename (whose own update already wrote+ * fresh tokens) is skipped. Runs across all tenants on the raw collections; the+ * `_id` filter keeps every write on its own document, and `tenantId` is never+ * written.+ */+export async function backfillSearchTokens(+ connection: Connection,+ options: { batchSize?: number; dryRun?: boolean } = {},+): Promise<SearchTokenBackfillResult> {+ const batchSize = Math.max(1, options.batchSize ?? DEFAULT_BATCH_SIZE);+ const result: SearchTokenBackfillResult = { pending: {}, updated: {} };++ for (const { name, fields } of SEARCH_TOKEN_COLLECTIONS) {+ const collection = connection.db!.collection(name);+ if (!options.dryRun) {+ /** Deployments running with `MONGO_AUTO_INDEX` off never build schema indexes, and+ * without these the token filters would scan the collection. Same specs and default+ * names as the schema declaration, so this is a no-op where Mongoose already built them. */+ for (const field of fields) {+ for (const index of searchTokenIndexes(field)) {+ await buildIndexWithRetry(+ () => collection.createIndex(index),+ `createIndex(${name}.${Object.keys(index).join('_')})`,+ );+ }+ }+ }+ result.pending[name] = 0;+ result.updated[name] = 0;++ const projection = Object.fromEntries(+ fields.flatMap((field) => [+ [field.source, 1],+ [field.tokens, 1],+ ]),+ );+ let batch: AnyBulkWriteOperation[] = [];+ const flush = async () => {+ if (batch.length === 0) {+ return;+ }+ // eslint-disable-next-line no-restricted-syntax -- offline all-tenant migration; `_id` filters keep each write on its own document+ const written = await collection.bulkWrite(batch, { ordered: false });+ result.updated[name] += written.modifiedCount;+ batch = [];+ };++ for await (const doc of collection.find({}, { projection })) {+ const expected = computeSearchTokenSet(fields, doc);+ if (fields.every((field) => sameTokens(doc[field.tokens], expected[field.tokens]))) {+ continue;+ }+ result.pending[name] += 1;+ if (options.dryRun) {+ continue;+ }+ const guard = Object.fromEntries(+ fields.map((field) => [field.source, doc[field.source] ?? null]),+ );+ batch.push({ updateOne: { filter: { _id: doc._id, ...guard }, update: { $set: expected } } });+ if (batch.length >= batchSize) {+ await flush();+ }+ }+ await flush();+ if (!options.dryRun) {+ logger.info(+ `[SearchTokenMigration] ${name}: ${result.updated[name]} of ${result.pending[name]} documents updated`,+ );+ }+ }+ return result;+}++/**+ * Logs a startup warning when users or groups still lack search tokens. Those+ * documents stay findable through the slower unindexed fallback until+ * `npm run migrate:search-tokens` runs. A best-effort diagnostic: a failed+ * check is logged and never blocks startup.+ *+ * Also warns when the token indexes are missing: with `MONGO_AUTO_INDEX` off,+ * a database whose documents all received tokens from the schema middleware+ * still never builds them until the backfill runs.+ *+ * The indexes are checked first, so the probe for a token-less document only+ * runs once they exist and it can seek them; `maxTimeMS` bounds it regardless.+ */+export async function warnOnMissingSearchTokens(connection: Connection): Promise<void> {+ try {+ const pending = await Promise.all(+ SEARCH_TOKEN_COLLECTIONS.map(async ({ name, fields }) => {+ const collection = connection.db!.collection(name);+ const indexes = await collection.indexes().catch((error: { codeName?: string }) => {+ /** No collection yet: nothing to search, nothing to migrate. */+ if (error?.codeName === 'NamespaceNotFound') {+ return null;+ }+ throw error;+ });+ if (indexes == null) {+ return null;+ }+ const built = new Set(indexes.map((index) => JSON.stringify(index.key)));+ const unindexed = fields.some((field) =>+ searchTokenIndexes(field).some((index) => !built.has(JSON.stringify(index))),+ );+ /** Without the indexes the probe below would scan; the warning is already due. */+ if (unindexed) {+ return name;+ }+ const unmigrated = await collection.findOne(missingTokens(fields), {+ projection: { _id: 1 },+ maxTimeMS: STARTUP_PROBE_MAX_TIME_MS,+ });+ return unmigrated != null ? name : null;+ }),+ );+ const names = pending.filter((name): name is string => name != null);+ if (names.length > 0) {+ logger.warn(+ `[SearchTokenMigration] Some ${names.join(' and ')} lack search tokens or their indexes; people search scans the collection until you run: npm run migrate:search-tokens`,+ );+ }+ } catch (error) {+ logger.error('[SearchTokenMigration] Failed to check search token migration:', error);+ }+}diff --git a/packages/data-schemas/src/models/plugins/searchTokens.spec.ts b/packages/data-schemas/src/models/plugins/searchTokens.spec.tsnew file mode 100644index 0000000000..edd1c6154d--- /dev/null+++ b/packages/data-schemas/src/models/plugins/searchTokens.spec.ts@@ -0,0 +1,318 @@+import mongoose from 'mongoose';+import { PrincipalType } from 'librechat-data-provider';+import { MongoMemoryServer } from 'mongodb-memory-server';+import type * as t from '~/types';+import { createUserGroupMethods } from '~/methods/userGroup';+import { tenantStorage } from '~/config/tenantContext';+import { buildUserSearchFilter } from '~/utils/search';+import { createModels } from '~/models';++jest.mock('~/config/winston', () => ({+ error: jest.fn(),+ warn: jest.fn(),+ info: jest.fn(),+ debug: jest.fn(),+}));++const TOKENS = '+nameTokens +emailTokens +usernameTokens';++let mongoServer: MongoMemoryServer;+let User: mongoose.Model<t.IUser>;+let Group: mongoose.Model<t.IGroup>;+let methods: ReturnType<typeof createUserGroupMethods>;++beforeAll(async () => {+ mongoServer = await MongoMemoryServer.create();+ await mongoose.connect(mongoServer.getUri());+ const models = createModels(mongoose);+ User = models.User;+ Group = models.Group as mongoose.Model<t.IGroup>;+ methods = createUserGroupMethods(mongoose);+});++afterAll(async () => {+ await mongoose.disconnect();+ await mongoServer.stop();+});++beforeEach(async () => {+ await mongoose.connection.dropDatabase();+});++const user = (name: string, email: string, username = '', extra: Partial<t.IUser> = {}) => ({+ name,+ email,+ username,+ provider: 'local',+ ...extra,+});++async function tokensOf(id: unknown) {+ return User.findById(id).select(TOKENS).lean<t.IUser>();+}++async function userNames(query: string, limit = 10): Promise<string[]> {+ const results = await methods.searchPrincipals(query, limit, [PrincipalType.USER]);+ return results.map((r) => r.name ?? '').sort();+}++describe('search token write paths', () => {+ it('derives tokens on create and on save of a changed name', async () => {+ const doc = await User.create(user('José Álvarez', '[email protected]', 'jalvarez'));+ expect(await tokensOf(doc._id)).toMatchObject({+ nameTokens: ['jose', 'alvarez'],+ emailTokens: ['[email protected]', 'jose', 'alvarez', 'example', 'org'],+ usernameTokens: ['jalvarez'],+ });++ doc.name = 'Joe Bloggs';+ await doc.save();+ const saved = await tokensOf(doc._id);+ expect(saved?.nameTokens).toEqual(['joe', 'bloggs']);+ expect(saved?.usernameTokens).toEqual(['jalvarez']);+ });++ it('updates tokens through findOneAndUpdate, findByIdAndUpdate, updateOne and updateMany', async () => {+ const [a, b] = await User.create([user('Ann A', '[email protected]'), user('Ben B', '[email protected]')]);++ await User.findOneAndUpdate({ _id: a._id }, { $set: { name: 'Carla Diaz' } });+ expect((await tokensOf(a._id))?.nameTokens).toEqual(['carla', 'diaz']);++ await User.findByIdAndUpdate(a._id, { email: '[email protected]' });+ expect((await tokensOf(a._id))?.emailTokens).toEqual(['[email protected]', 'carla', 'new', 'dev']);++ await User.updateOne({ _id: b._id }, { $set: { username: 'ben_b' } });+ expect((await tokensOf(b._id))?.usernameTokens).toEqual(['ben_b', 'ben', 'b']);++ await User.updateMany({}, { $unset: { name: '' } });+ expect((await tokensOf(a._id))?.nameTokens).toEqual([]);+ expect((await tokensOf(b._id))?.nameTokens).toEqual([]);+ });++ it('sets tokens on upsert inserts and insertMany', async () => {+ await User.updateOne(+ { email: '[email protected]' },+ { $setOnInsert: { name: 'New Person', email: '[email protected]', provider: 'local' } },+ { upsert: true },+ );+ const upserted = await User.findOne({ email: '[email protected]' }).select(TOKENS).lean<t.IUser>();+ expect(upserted?.nameTokens).toEqual(['new', 'person']);+ expect(upserted?.usernameTokens).toEqual([]);++ await User.updateOne(+ { email: '[email protected]' },+ { $set: { name: 'From Filter', provider: 'local' } },+ { upsert: true },+ );+ const fromFilter = await User.findOne({ email: '[email protected]' }).select(TOKENS).lean<t.IUser>();+ expect(fromFilter?.emailTokens).toEqual(['[email protected]', 'filter', 'x', 'io']);++ await User.insertMany([user('Many One', '[email protected]', 'm1')]);+ const inserted = await User.findOne({ email: '[email protected]' }).select(TOKENS).lean<t.IUser>();+ expect(inserted?.nameTokens).toEqual(['many', 'one']);+ expect(inserted?.usernameTokens).toEqual(['m1']);+ });++ it('keeps tokens out of default reads', async () => {+ const doc = await User.create(user('Hidden Tokens', '[email protected]'));+ const plain = await User.findById(doc._id).lean<t.IUser>();+ expect(plain).not.toHaveProperty('nameTokens');+ });++ it('updates group tokens through create, upsertGroupByExternalId and updateGroupById', async () => {+ const group = await methods.createGroup({ name: 'Core Platform', source: 'local' });+ const read = () =>+ Group.findById(group._id).select('+nameTokens +emailTokens').lean<t.IGroup>();+ expect((await read())?.nameTokens).toEqual(['core', 'platform']);++ await methods.updateGroupById(group._id, { name: 'Data Science', email: '[email protected]' });+ expect(await read()).toMatchObject({+ nameTokens: ['data', 'science'],+ emailTokens: ['[email protected]', 'ds', 'corp', 'io'],+ });++ const entra = await methods.upsertGroupByExternalId('ext-1', 'entra', { name: 'Sales EMEA' });+ const entraTokens = await Group.findById(entra?._id).select('+nameTokens').lean<t.IGroup>();+ expect(entraTokens?.nameTokens).toEqual(['sales', 'emea']);+ expect(await Group.countDocuments({ _id: entra?._id, emailTokens: { $exists: true } })).toBe(1);+ });++ it('keeps tokens when an update assigns undefined to a source field', async () => {+ const doc = await User.create(user('Kept Name', '[email protected]'));+ await User.findByIdAndUpdate(doc._id, { $set: { name: undefined, role: 'ADMIN' } });+ const stored = await User.findById(doc._id).select(`name ${TOKENS}`).lean<t.IUser>();+ expect(stored?.name).toBe('Kept Name');+ expect(stored?.nameTokens).toEqual(['kept', 'name']);+ });+});++describe('searchPrincipals word-prefix search', () => {+ beforeEach(async () => {+ await User.create([+ user('John Smith', '[email protected]', 'jsmith'),+ user('Johanna Müller', '[email protected]', 'hanna_m'),+ user('Bob Johnson', '[email protected]', 'bobby'),+ user('Zoë Ng', '[email protected]', 'zng'),+ ]);+ });++ it('matches the prefix of a first name, a last name, an email word, a domain and a username', async () => {+ expect(await userNames('joh')).toEqual(['Bob Johnson', 'Johanna Müller', 'John Smith']);+ expect(await userNames('smi')).toEqual(['John Smith']);+ expect(await userNames('mull')).toEqual(['Johanna Müller']);+ expect(await userNames('acme')).toEqual(['John Smith', 'Zoë Ng']);+ expect(await userNames('hanna')).toEqual(['Johanna Müller']);+ expect(await userNames('john.smith@ac')).toEqual(['John Smith']);+ });++ it('requires every query token, across fields', async () => {+ expect(await userNames('john smi')).toEqual(['John Smith']);+ expect(await userNames('smith john')).toEqual(['John Smith']);+ expect(await userNames('jo acme')).toEqual(['John Smith']);+ expect(await userNames('john globex')).toEqual([]);+ });++ it('is case- and accent-insensitive', async () => {+ expect(await userNames('ZOE')).toEqual(['Zoë Ng']);+ expect(await userNames('MÜLLER')).toEqual(['Johanna Müller']);+ });++ it('does not match inside a word (intended: only word prefixes are index-bound)', async () => {+ expect(await userNames('ohn')).toEqual([]);+ expect(await userNames('mith')).toEqual([]);+ });++ it('respects the limit', async () => {+ expect(await userNames('jo', 2)).toHaveLength(2);+ });++ it('finds documents written before tokens existed through the substring fallback', async () => {+ await User.collection.insertOne({ name: 'Legacy Person', email: '[email protected]' });+ expect(await userNames('gacy')).toEqual(['Legacy Person']);+ });++ it('stays inside the active tenant', async () => {+ await tenantStorage.run({ tenantId: 'tenant-a' }, () =>+ User.create(user('John Tenant', '[email protected]')),+ );+ await tenantStorage.run({ tenantId: 'tenant-b' }, () =>+ User.create(user('John Other', '[email protected]')),+ );+ const names = await tenantStorage.run({ tenantId: 'tenant-a' }, () => userNames('john'));+ expect(names).toEqual(['John Tenant']);+ });++ it('finds groups by name and email word prefix', async () => {+ await Group.create([+ { name: 'Platform Team', source: 'local', email: '[email protected]' },+ { name: 'Design', source: 'local', description: 'Platform design' },+ ]);+ const groups = await methods.searchPrincipals('plat', 10, [PrincipalType.GROUP]);+ expect(groups.map((g) => g.name)).toEqual(['Platform Team']);+ });+});++describe('search token serialization', () => {+ it('keeps token fields out of the JSON and object output of a freshly created document', async () => {+ const [group] = await Group.create([{ name: 'Platform Team', source: 'local' }]);+ const created = await User.create(user('Ana Lima', '[email protected]', 'ana'));+ for (const doc of [group, created]) {+ expect(doc.get('nameTokens')).toBeDefined();+ expect(doc.toJSON()).not.toHaveProperty('nameTokens');+ expect(doc.toObject()).not.toHaveProperty('nameTokens');+ expect(doc.toObject()).not.toHaveProperty('emailTokens');+ }+ const stored = await Group.findById(group._id).select('+nameTokens').lean();+ expect(stored?.nameTokens).toEqual(['platform', 'team']);+ });+});++describe('search query plan', () => {+ const planStages = (plan: unknown): string[] => {+ const stages: string[] = [];+ const visit = (node: unknown) => {+ if (node == null || typeof node !== 'object') {+ return;+ }+ const record = node as Record<string, unknown>;+ if (typeof record.stage === 'string') {+ stages.push(record.stage);+ }+ for (const value of Object.values(record)) {+ visit(value);+ }+ };+ visit(plan);+ return stages;+ };++ beforeEach(async () => {+ await User.createIndexes();+ await User.create([+ user('John Smith', '[email protected]', 'jsmith'),+ user('Bob Johnson', '[email protected]', 'bobby'),+ ]);+ });++ it.each(['john', 'john smi', 'john.smith@acme'])(+ 'uses index scans, never a collection scan, for "%s"',+ async (query) => {+ const filter = buildUserSearchFilter(query)!;+ const explain = (await User.find(filter).limit(10).explain('queryPlanner')) as unknown as {+ queryPlanner: { winningPlan: unknown };+ };+ const stages = planStages(explain.queryPlanner.winningPlan);+ expect(stages).toContain('IXSCAN');+ expect(stages).not.toContain('COLLSCAN');+ },+ );++ it('uses index scans with the admin search sort by name', async () => {+ const filter = buildUserSearchFilter('jo')!;+ const explain = (await User.find(filter)+ .sort({ name: 1 })+ .limit(20)+ .explain('queryPlanner')) as unknown as { queryPlanner: { winningPlan: unknown } };+ const stages = planStages(explain.queryPlanner.winningPlan);+ expect(stages).toContain('IXSCAN');+ expect(stages).not.toContain('COLLSCAN');+ });++ it('uses index scans under a tenant scope', async () => {+ const filter = buildUserSearchFilter('john')!;+ const explain = (await tenantStorage.run({ tenantId: 'tenant-a' }, () =>+ User.find(filter).limit(10).explain('queryPlanner').exec(),+ )) as unknown as { queryPlanner: { winningPlan: unknown } };+ const stages = planStages(explain.queryPlanner.winningPlan);+ expect(stages).toContain('IXSCAN');+ expect(stages).not.toContain('COLLSCAN');+ });++ it('seeks only the scoped tenant keys when every tenant shares the prefix', async () => {+ /** The scoped tenant is large with two matches; every other tenant is all matches,+ * so neither a tenant-only nor a token-first seek can stay small. */+ const doc = (name: string, i: number, tenant: string) => ({+ name: `${name} ${i}`,+ email: `${name.toLowerCase()}${i}@${tenant}.io`,+ tenantId: tenant,+ nameTokens: [name.toLowerCase(), String(i)],+ emailTokens: [+ `${name.toLowerCase()}${i}@${tenant}.io`,+ `${name.toLowerCase()}${i}`,+ tenant,+ 'io',+ ],+ usernameTokens: [],+ });+ await User.collection.insertMany([+ ...Array.from({ length: 300 }, (_, i) => doc(i < 2 ? 'Johnny' : 'Anna', i, 'tenant-3')),+ ...Array.from({ length: 300 }, (_, i) => doc('Johnny', i, `tenant-${10 + (i % 10)}`)),+ ]);+ const filter = buildUserSearchFilter('johnny')!;+ const explain = (await tenantStorage.run({ tenantId: 'tenant-3' }, () =>+ User.find(filter).explain('executionStats').exec(),+ )) as unknown as { executionStats: { nReturned: number; totalKeysExamined: number } };+ expect(explain.executionStats.nReturned).toBe(2);+ expect(explain.executionStats.totalKeysExamined).toBeLessThanOrEqual(10);+ });+});diff --git a/packages/data-schemas/src/models/plugins/searchTokens.ts b/packages/data-schemas/src/models/plugins/searchTokens.tsnew file mode 100644index 0000000000..67b935b57b--- /dev/null+++ b/packages/data-schemas/src/models/plugins/searchTokens.ts@@ -0,0 +1,71 @@+import type { Schema, Query } from 'mongoose';+import type { SearchTokenField } from '~/utils/search';+import { withSearchTokens, searchTokenIndexes, computeSearchTokens } from '~/utils/search';++/**+ * Adds the derived search-token arrays of `fields` to `schema`, indexes each+ * with `tenantId` in both key orders (see `searchTokenIndexes`), and keeps them in sync on `save`, `insertMany` and the+ * update queries. `bulkWrite` runs no middleware: callers that change a source+ * field through it must apply `withSearchTokens` themselves.+ *+ * Token fields are `select: false` and stripped from `toJSON`/`toObject`+ * output (a freshly saved document still holds them); they exist for the+ * index, not for readers.+ */+export function applySearchTokens(schema: Schema, fields: readonly SearchTokenField[]): void {+ for (const field of fields) {+ schema.add({ [field.tokens]: { type: [String], default: undefined, select: false } });+ for (const index of searchTokenIndexes(field)) {+ schema.index(index);+ }+ }++ for (const option of ['toJSON', 'toObject'] as const) {+ schema.set(option, {+ ...schema.get(option),+ transform(_doc, ret: Record<string, unknown>) {+ for (const field of fields) {+ delete ret[field.tokens];+ }+ return ret;+ },+ });+ }++ schema.pre('save', function () {+ for (const field of fields) {+ if (this.isNew || this.isModified(field.source)) {+ this.set(field.tokens, computeSearchTokens(field.kind, this.get(field.source)));+ }+ }+ });++ schema.pre(+ ['findOneAndUpdate', 'updateOne', 'updateMany'],+ { document: false, query: true },+ function (this: Query<unknown, unknown>) {+ const update = this.getUpdate();+ if (!update) {+ return;+ }+ const next = withSearchTokens(fields, update, {+ upsert: this.getOptions().upsert === true,+ filter: this.getFilter(),+ });+ if (next !== update) {+ this.setUpdate(next);+ }+ },+ );++ schema.pre('insertMany', function (next, docs: unknown) {+ if (Array.isArray(docs)) {+ for (const doc of docs as Array<Record<string, unknown>>) {+ for (const field of fields) {+ doc[field.tokens] = computeSearchTokens(field.kind, doc[field.source]);+ }+ }+ }+ next();+ });+}diff --git a/packages/data-schemas/src/schema/group.ts b/packages/data-schemas/src/schema/group.tsindex bb19e4aa81..17c5e1bd06 100644--- a/packages/data-schemas/src/schema/group.ts+++ b/packages/data-schemas/src/schema/group.ts@@ -1,5 +1,7 @@import { Schema } from 'mongoose';import type { IGroup } from '~/types';+import { applySearchTokens } from '~/models/plugins/searchTokens';+import { GROUP_SEARCH_TOKEN_FIELDS } from '~/utils/search';const groupSchema: Schema<IGroup> = new Schema<IGroup>({@@ -57,5 +59,6 @@ groupSchema.index(},);groupSchema.index({ memberIds: 1, tenantId: 1 });+applySearchTokens(groupSchema, GROUP_SEARCH_TOKEN_FIELDS);export default groupSchema;diff --git a/packages/data-schemas/src/schema/user.ts b/packages/data-schemas/src/schema/user.tsindex 87ba658ce0..0b7329d322 100644--- a/packages/data-schemas/src/schema/user.ts+++ b/packages/data-schemas/src/schema/user.ts@@ -1,5 +1,7 @@import { Schema } from 'mongoose';import { SystemRoles, STATEFUL_CODE_ENVIRONMENTS } from 'librechat-data-provider';+import { applySearchTokens } from '~/models/plugins/searchTokens';+import { USER_SEARCH_TOKEN_FIELDS } from '~/utils/search';import { IUser } from '~/types';// Session sub-schema@@ -244,6 +246,7 @@ userSchema.index({ idOnTheSource: 1, openidIssuer: 1, tenantId: 1 });them. The rare cleanup that pulls a deleted agent from every user names the tenant too;the one that does not (`$in` over a batch of ids) is a maintenance write, not a page. */userSchema.index({ tenantId: 1, 'favorites.agentId': 1 });+applySearchTokens(userSchema, USER_SEARCH_TOKEN_FIELDS);const oAuthIdFields = ['googleId',diff --git a/packages/data-schemas/src/types/group.ts b/packages/data-schemas/src/types/group.tsindex 15cb6f288e..f935b9837f 100644--- a/packages/data-schemas/src/types/group.ts+++ b/packages/data-schemas/src/types/group.ts@@ -15,6 +15,9 @@ export interface IGroup extends Document {createdAt?: Date;updatedAt?: Date;tenantId?: string;+ /** Derived search tokens (see `~/utils/search`); `select: false`. */+ nameTokens?: string[];+ emailTokens?: string[];}export interface CreateGroupRequest {diff --git a/packages/data-schemas/src/types/user.ts b/packages/data-schemas/src/types/user.tsindex e2b3cd861b..9b0e8837b8 100644--- a/packages/data-schemas/src/types/user.ts+++ b/packages/data-schemas/src/types/user.ts@@ -89,6 +89,10 @@ export interface IUser extends Document {/** Field for external source identification (for consistency with TPrincipal schema) */idOnTheSource?: string;tenantId?: string;+ /** Derived search tokens (see `~/utils/search`); `select: false`. */+ nameTokens?: string[];+ emailTokens?: string[];+ usernameTokens?: string[];federatedTokens?: OIDCTokens;openidTokens?: OIDCTokens;}diff --git a/packages/data-schemas/src/utils/index.ts b/packages/data-schemas/src/utils/index.tsindex f5a4c6fd48..828eb80429 100644--- a/packages/data-schemas/src/utils/index.ts+++ b/packages/data-schemas/src/utils/index.ts@@ -1,5 +1,6 @@export * from './principal';export * from './string';+export * from './search';export * from './tempChatRetention';export * from './retention';export { tenantSafeBulkWrite } from './tenantBulkWrite';diff --git a/packages/data-schemas/src/utils/search.spec.ts b/packages/data-schemas/src/utils/search.spec.tsnew file mode 100644index 0000000000..db38aebcb6--- /dev/null+++ b/packages/data-schemas/src/utils/search.spec.ts@@ -0,0 +1,215 @@+import {+ withSearchTokens,+ computeSearchTokens,+ normalizeSearchText,+ buildUserSearchFilter,+ MAX_SEARCH_TOKENS,+ USER_SEARCH_TOKEN_FIELDS,+ MAX_SEARCH_TOKEN_LENGTH,+ MAX_SEARCH_QUERY_TOKENS,+} from './search';++describe('normalizeSearchText', () => {+ it('lowercases and folds accents and compatibility forms', () => {+ expect(normalizeSearchText('José ÅNGSTRÖM')).toBe('jose angstrom');+ expect(normalizeSearchText('Full')).toBe('full');+ expect(normalizeSearchText('İstanbul')).toBe('istanbul');+ });+});++describe('computeSearchTokens', () => {+ it('splits names on whitespace and punctuation', () => {+ expect(computeSearchTokens('words', " Mary-Jane O'Brien, Jr. ")).toEqual([+ 'mary',+ 'jane',+ 'o',+ 'brien',+ 'jr',+ ]);+ });++ it('keeps non-Latin scripts as words', () => {+ expect(computeSearchTokens('words', '山田 太郎')).toEqual(['山田', '太郎']);+ expect(computeSearchTokens('words', 'Ζωή Κ.')).toEqual(['ζωη', 'κ']);+ });++ it('indexes the full email, local-part words, and domain labels', () => {+ expect(computeSearchTokens('email', '[email protected]')).toEqual([+ '[email protected]',+ 'john',+ 'smith',+ 'work',+ 'mail',+ 'example',+ 'co',+ 'com',+ 'example-co',+ ]);+ });++ it('indexes the full username and its parts', () => {+ expect(computeSearchTokens('handle', 'jane_doe.99')).toEqual([+ 'jane_doe.99',+ 'jane',+ 'doe',+ '99',+ ]);+ });++ it('dedupes and drops empties', () => {+ expect(computeSearchTokens('words', 'Ann ann ANN --')).toEqual(['ann']);+ expect(computeSearchTokens('words', ' ')).toEqual([]);+ expect(computeSearchTokens('words', '')).toEqual([]);+ });++ it('returns no tokens for non-strings', () => {+ expect(computeSearchTokens('words', undefined)).toEqual([]);+ expect(computeSearchTokens('email', null)).toEqual([]);+ expect(computeSearchTokens('handle', 42)).toEqual([]);+ });++ it('caps token length and token count', () => {+ const long = 'a'.repeat(MAX_SEARCH_TOKEN_LENGTH + 10);+ expect(computeSearchTokens('words', long)).toEqual(['a'.repeat(MAX_SEARCH_TOKEN_LENGTH)]);++ const many = Array.from({ length: MAX_SEARCH_TOKENS + 10 }, (_, i) => `w${i}`).join(' ');+ const tokens = computeSearchTokens('words', many);+ expect(tokens).toHaveLength(MAX_SEARCH_TOKENS);+ expect(tokens[0]).toBe('w0');+ });++ it('does not split a surrogate pair when truncating', () => {+ const [token] = computeSearchTokens('words', '\u{10428}'.repeat(MAX_SEARCH_TOKEN_LENGTH + 5));+ expect(Array.from(token)).toHaveLength(MAX_SEARCH_TOKEN_LENGTH);+ expect(token).toBe('\u{10428}'.repeat(MAX_SEARCH_TOKEN_LENGTH));+ });+});++describe('withSearchTokens', () => {+ it('derives tokens for each source field in $set and leaves the input untouched', () => {+ const update = { $set: { name: 'Ana Lima', role: 'USER' } };+ const next = withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update);+ expect(next).toEqual({ $set: { name: 'Ana Lima', role: 'USER', nameTokens: ['ana', 'lima'] } });+ expect(update).toEqual({ $set: { name: 'Ana Lima', role: 'USER' } });+ });++ it('handles top-level, $setOnInsert and $unset writes', () => {+ expect(+ withSearchTokens(USER_SEARCH_TOKEN_FIELDS, {+ username: 'al',+ $setOnInsert: { email: '[email protected]' },+ $unset: { name: '' },+ }),+ ).toEqual({+ username: 'al',+ usernameTokens: ['al'],+ $setOnInsert: { email: '[email protected]', emailTokens: ['[email protected]', 'a', 'b', 'io'] },+ $unset: { name: '' },+ $set: { nameTokens: [] },+ });+ });++ it('ignores undefined assignments, which Mongoose drops from the update', () => {+ const update = { $set: { name: undefined, role: 'ADMIN' }, username: undefined };+ expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update);+ });++ it('ignores an undefined $unset, which Mongoose also drops', () => {+ const update = { $unset: { name: undefined } };+ expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update);+ });++ it('initializes every unwritten token field on upsert inserts', () => {+ expect(+ withSearchTokens(USER_SEARCH_TOKEN_FIELDS, { $set: { name: 'Al' } }, { upsert: true }),+ ).toEqual({+ $set: { name: 'Al', nameTokens: ['al'] },+ $setOnInsert: { emailTokens: [], usernameTokens: [] },+ });+ });++ it('takes upsert insert tokens from equality conditions the insert copies', () => {+ expect(+ withSearchTokens(+ USER_SEARCH_TOKEN_FIELDS,+ { $set: { name: 'Al' } },+ { upsert: true, filter: { email: '[email protected]', username: { $eq: 'al' } } },+ ),+ ).toEqual({+ $set: { name: 'Al', nameTokens: ['al'] },+ $setOnInsert: { emailTokens: ['[email protected]', 'al', 'x', 'io'], usernameTokens: ['al'] },+ });+ expect(+ withSearchTokens(+ USER_SEARCH_TOKEN_FIELDS,+ { $set: { name: 'Al' } },+ { upsert: true, filter: { email: { $in: ['[email protected]', '[email protected]'] } } },+ ),+ ).toEqual({+ $set: { name: 'Al', nameTokens: ['al'] },+ $setOnInsert: { usernameTokens: [] },+ });+ });++ it('returns the same reference when no source field is written', () => {+ const update = { $set: { role: 'ADMIN' } };+ expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, update)).toBe(update);+ const pipeline = [{ $set: { name: 'x' } }];+ expect(withSearchTokens(USER_SEARCH_TOKEN_FIELDS, pipeline)).toBe(pipeline);+ });+});++describe('buildUserSearchFilter', () => {+ it('returns null when the query has nothing to match', () => {+ expect(buildUserSearchFilter('')).toBeNull();+ expect(buildUserSearchFilter(' ')).toBeNull();+ });++ it('matches a single token by anchored, case-sensitive prefix on every token field', () => {+ const filter = buildUserSearchFilter(' José ') as { $or: Array<Record<string, unknown>> };+ const prefixes = filter.$or.filter((branch) => Object.keys(branch).length === 1);+ expect(prefixes).toEqual([+ { nameTokens: /^jose/ },+ { emailTokens: /^jose/ },+ { usernameTokens: /^jose/ },+ ]);+ expect(filter.$or).toContainEqual({ nameTokens: { $exists: false }, name: /José/i });+ });++ it('anchors the legacy fallback for callers whose old search was a prefix match', () => {+ const filter = buildUserSearchFilter('mith', { legacyPrefix: true })!;+ expect(filter.$or).toContainEqual({ nameTokens: { $exists: false }, name: /^mith/i });+ expect(filter.$or).not.toContainEqual({ nameTokens: { $exists: false }, name: /mith/i });+ });++ it('requires every query token and also tries the whole query on full-value fields', () => {+ const filter = buildUserSearchFilter('john smi') as { $and: Array<{ $or: unknown[] }> };+ expect(filter.$and).toHaveLength(2);+ expect(filter.$and[0].$or).toContainEqual({ nameTokens: /^john/ });+ expect(filter.$and[1].$or).toContainEqual({ nameTokens: /^smi/ });+ expect(filter.$and[1].$or).toContainEqual({ emailTokens: /^john smi/ });+ expect(filter.$and[1].$or).not.toContainEqual({ nameTokens: /^john smi/ });+ });++ it('escapes regex metacharacters and requires every query token up to the cap', () => {+ const filter = buildUserSearchFilter('a.b') as { $and: Array<{ $or: unknown[] }> };+ expect(filter.$and[0].$or).toContainEqual({ emailTokens: /^a\.b/ });++ const words = (n: number) => Array.from({ length: n }, (_, i) => `t${i}`).join(' ');+ const all = buildUserSearchFilter(words(MAX_SEARCH_QUERY_TOKENS)) as {+ $and: Array<{ $or: unknown[] }>;+ };+ expect(all.$and).toHaveLength(MAX_SEARCH_QUERY_TOKENS);+ expect(all.$and[MAX_SEARCH_QUERY_TOKENS - 1].$or).toContainEqual({+ nameTokens: new RegExp(`^t${MAX_SEARCH_QUERY_TOKENS - 1}`),+ });+ /** Over the cap, nothing matches: extra words are never silently dropped. */+ expect(buildUserSearchFilter(words(MAX_SEARCH_QUERY_TOKENS + 1))).toBeNull();+ });++ it('searches punctuation-only queries on full values only', () => {+ const filter = buildUserSearchFilter('.*') as { $or: unknown[] };+ expect(filter.$or).toContainEqual({ emailTokens: /^\.\*/ });+ expect(filter.$or).not.toContainEqual({ nameTokens: expect.anything() });+ });+});diff --git a/packages/data-schemas/src/utils/search.ts b/packages/data-schemas/src/utils/search.tsnew file mode 100644index 0000000000..63a232561d--- /dev/null+++ b/packages/data-schemas/src/utils/search.ts@@ -0,0 +1,328 @@+import { escapeRegExp } from './string';++/**+ * Word-prefix search over derived token arrays.+ *+ * Each searchable source field (`name`, `email`, `username`) gets a lowercase,+ * accent-folded token array stored beside it and indexed with `tenantId`. A query+ * is tokenized the same way, and each query token must prefix-match some token,+ * so "john smi" finds "John Smith" through an anchored, case-sensitive regex+ * that the multikey index can bound. Matches inside a word ("ohn") are not+ * found, by design: that is the price of an index-bound search.+ *+ * The token rules are a stored format: changing them requires re-running the+ * backfill, so the caps are constants rather than configuration.+ */++/** How a source value is split into tokens. */+export type SearchTokenKind = 'words' | 'email' | 'handle';++export interface SearchTokenField {+ /** The stored field the tokens derive from. */+ readonly source: string;+ /** The derived, indexed token array. */+ readonly tokens: string;+ readonly kind: SearchTokenKind;+}++export const USER_SEARCH_TOKEN_FIELDS: readonly SearchTokenField[] = [+ { source: 'name', tokens: 'nameTokens', kind: 'words' },+ { source: 'email', tokens: 'emailTokens', kind: 'email' },+ { source: 'username', tokens: 'usernameTokens', kind: 'handle' },+];++export const GROUP_SEARCH_TOKEN_FIELDS: readonly SearchTokenField[] = [+ { source: 'name', tokens: 'nameTokens', kind: 'words' },+ { source: 'email', tokens: 'emailTokens', kind: 'email' },+];++/** Longest stored or queried word token; longer words are truncated to this prefix. */+export const MAX_SEARCH_TOKEN_LENGTH = 64;+/** Longest whole-value token (a full email address or username). */+export const MAX_SEARCH_VALUE_LENGTH = 256;+/** Most tokens stored per field. */+export const MAX_SEARCH_TOKENS = 32;+/**+ * Most distinct query words. Every word must match, so a longer query matches+ * nothing rather than ignoring its extra words; the cap bounds the filter a+ * single request can build.+ */+export const MAX_SEARCH_QUERY_TOKENS = 16;++const NON_WORD = /[^\p{L}\p{N}]+/u;+const COMBINING_MARKS = /\p{M}+/gu;++function truncate(value: string, max: number): string {+ if (value.length <= max) {+ return value;+ }+ return Array.from(value).slice(0, max).join('');+}++/** Lowercases and folds accents, so "José" and "jose" compare equal. */+export function normalizeSearchText(value: string): string {+ return value.normalize('NFKD').replace(COMBINING_MARKS, '').toLowerCase();+}++function words(normalized: string): string[] {+ return normalized+ .split(NON_WORD)+ .filter(Boolean)+ .map((word) => truncate(word, MAX_SEARCH_TOKEN_LENGTH));+}++/**+ * A value reduced to the words the filter matches, joined by single spaces, so+ * relevance scoring sees "Mary-Jane" and "mary jane" as the same text.+ */+export function normalizeSearchPhrase(value: string): string {+ return words(normalizeSearchText(value)).join(' ');+}++/**+ * Relevance of a matched document for `query`, over the values the filter+ * searched. Like the filter, it ignores word order: 100 when a value has the+ * query's words in any order, 80 when a value starts with the query, 50 when a+ * value contains it or every query word prefixes some word (in one value or+ * across them), 10 otherwise.+ */+export function scoreSearchMatch(+ values: ReadonlyArray<string | null | undefined>,+ query: string,+): number {+ const phrase = normalizeSearchPhrase(query);+ if (!phrase) {+ return 10;+ }+ const queryWords = phrase.split(' ');+ const sortedQuery = [...queryWords].sort().join(' ');+ const valueWords: string[] = [];+ let score = 10;+ for (const value of values) {+ if (!value) {+ continue;+ }+ const field = normalizeSearchPhrase(value);+ const words = field.split(' ');+ if (field === phrase || [...words].sort().join(' ') === sortedQuery) {+ return 100;+ }+ if (field.startsWith(phrase)) {+ score = Math.max(score, 80);+ } else if (field.includes(phrase)) {+ score = Math.max(score, 50);+ }+ valueWords.push(...words);+ }+ const everyWord = queryWords.every((queryWord) =>+ valueWords.some((word) => word.startsWith(queryWord)),+ );+ return everyWord ? Math.max(score, 50) : score;+}++/** Derives the stored token array for one field value. Non-strings yield no tokens. */+export function computeSearchTokens(kind: SearchTokenKind, value: unknown): string[] {+ if (typeof value !== 'string') {+ return [];+ }+ const normalized = normalizeSearchText(value.trim());+ if (!normalized) {+ return [];+ }+ const tokens: string[] = [];+ if (kind !== 'words') {+ tokens.push(truncate(normalized, MAX_SEARCH_VALUE_LENGTH));+ }+ tokens.push(...words(normalized));+ if (kind === 'email') {+ const domain = normalized.slice(normalized.lastIndexOf('@') + 1);+ tokens.push(...domain.split('.').filter(Boolean));+ }+ return [...new Set(tokens)].slice(0, MAX_SEARCH_TOKENS);+}++/** Every token field of `fields`, computed from a document's source values. */+export function computeSearchTokenSet(+ fields: readonly SearchTokenField[],+ doc: Record<string, unknown>,+): Record<string, string[]> {+ const result: Record<string, string[]> = {};+ for (const field of fields) {+ result[field.tokens] = computeSearchTokens(field.kind, doc[field.source]);+ }+ return result;+}++type UpdateDoc = Record<string, unknown>;++/**+ * The indexes behind each token field. Tenant first serves tenant-scoped+ * searches with an equality seek before the prefix range; token first serves+ * unscoped ones (single-tenant deployments run without a tenant context), which+ * could not use the tenant-first index at all.+ */+export function searchTokenIndexes(field: SearchTokenField): Array<Record<string, 1>> {+ return [+ { tenantId: 1, [field.tokens]: 1 },+ { [field.tokens]: 1, tenantId: 1 },+ ];+}++/** Mongoose drops `undefined` assignments from an update, so they write nothing. */+function writes(doc: UpdateDoc | undefined, key: string): doc is UpdateDoc {+ return doc != null && key in doc && doc[key] !== undefined;+}++const SKIP = Symbol('skip');++/**+ * The value an upsert insert takes for `key` from the query filter: a plain or+ * `$eq` equality is copied into the new document, no condition leaves it unset.+ * Any other condition is ambiguous, so the token field is left for the backfill.+ */+function insertedValue(filter: UpdateDoc | undefined, key: string): unknown {+ const condition = filter?.[key];+ if (condition === undefined) {+ return undefined;+ }+ if (typeof condition === 'string') {+ return condition;+ }+ const eq = operator({ condition }, 'condition');+ return eq && Object.keys(eq).length === 1 && typeof eq.$eq === 'string' ? eq.$eq : SKIP;+}++function operator(update: UpdateDoc, key: string): UpdateDoc | undefined {+ const value = update[key];+ return value != null && typeof value === 'object' && !Array.isArray(value)+ ? (value as UpdateDoc)+ : undefined;+}++/**+ * Returns `update` with the token fields recomputed for every source field it+ * writes, through `$set`, `$setOnInsert`, `$unset` or a top-level replacement+ * value. Each token field depends on one source field, so a partial update+ * stays correct. On an upsert, a token field whose source the update does not+ * write is initialized on insert from the source's equality condition in+ * `filter` (MongoDB copies it into the new document), or to `[]` when the+ * filter has none, so new documents never look un-migrated. Copy-on-write: the same reference comes back when nothing+ * changes. Pipeline updates are returned untouched.+ *+ * Mongoose runs no middleware for `bulkWrite`; a bulk write that changes a+ * source field must pass its update through this function.+ */+export function withSearchTokens<T>(+ fields: readonly SearchTokenField[],+ update: T,+ options: { upsert?: boolean; filter?: UpdateDoc } = {},+): T {+ if (update == null || typeof update !== 'object' || Array.isArray(update)) {+ return update;+ }+ const source = update as UpdateDoc;+ let next: UpdateDoc | undefined;+ const write = (op: string | null, key: string, tokens: string[]) => {+ next ??= { ...source };+ if (op == null) {+ next[key] = tokens;+ return;+ }+ next[op] = { ...(operator(next, op) ?? {}), [key]: tokens };+ };++ for (const field of fields) {+ const set = operator(source, '$set');+ const setOnInsert = operator(source, '$setOnInsert');+ const unset = operator(source, '$unset');+ let written = true;+ if (writes(set, field.source)) {+ write('$set', field.tokens, computeSearchTokens(field.kind, set[field.source]));+ } else if (writes(source, field.source)) {+ write(null, field.tokens, computeSearchTokens(field.kind, source[field.source]));+ } else if (writes(unset, field.source)) {+ write('$set', field.tokens, []);+ } else {+ written = false;+ }+ if (writes(setOnInsert, field.source)) {+ write(+ '$setOnInsert',+ field.tokens,+ computeSearchTokens(field.kind, setOnInsert[field.source]),+ );+ } else if (options.upsert && !written) {+ const condition = insertedValue(options.filter, field.source);+ if (condition !== SKIP) {+ write('$setOnInsert', field.tokens, computeSearchTokens(field.kind, condition));+ }+ }+ }+ return (next ?? source) as T;+}++export interface SearchFilterOptions {+ /** Anchor the legacy fallback at the start of the field, for callers whose old search was a prefix match. */+ legacyPrefix?: boolean;+}++/**+ * Builds the filter for a word-prefix search, or `null` when the query has+ * nothing to match.+ *+ * Every query token must prefix-match a token of some field (fields may+ * differ per token, so "john gmail" finds John at gmail.com). Fields that+ * store their whole value (email, username) also match the whole query as a+ * prefix, so a pasted "john.smith@exa" works. Documents written before the+ * token fields existed fall back to the caller's old case-insensitive regex+ * on the source field (unanchored, or anchored with `legacyPrefix`); the+ * fallback branches seek the token index on `$exists: false` and match+ * nothing once the backfill has run.+ */+export function buildSearchTokenFilter(+ fields: readonly SearchTokenField[],+ query: string,+ options: SearchFilterOptions = {},+): Record<string, unknown> | null {+ const trimmed = query.trim();+ const normalized = normalizeSearchText(trimmed);+ const queryTokens = [...new Set(words(normalized))];+ const whole = truncate(normalized, MAX_SEARCH_VALUE_LENGTH);+ const matchWhole = whole.length > 0 && !(queryTokens.length === 1 && queryTokens[0] === whole);+ if (+ !trimmed ||+ queryTokens.length > MAX_SEARCH_QUERY_TOKENS ||+ (queryTokens.length === 0 && !matchWhole)+ ) {+ return null;+ }++ const legacy = new RegExp(`${options.legacyPrefix ? '^' : ''}${escapeRegExp(trimmed)}`, 'i');+ const shared: Record<string, unknown>[] = [];+ for (const field of fields) {+ if (matchWhole && field.kind !== 'words') {+ shared.push({ [field.tokens]: new RegExp(`^${escapeRegExp(whole)}`) });+ }+ shared.push({ [field.tokens]: { $exists: false }, [field.source]: legacy });+ }++ if (queryTokens.length === 0) {+ return { $or: shared };+ }+ const clauses = queryTokens.map((token) => {+ const prefix = new RegExp(`^${escapeRegExp(token)}`);+ return { $or: [...fields.map((field) => ({ [field.tokens]: prefix })), ...shared] };+ });+ return clauses.length === 1 ? clauses[0] : { $and: clauses };+}++export function buildUserSearchFilter(+ query: string,+ options?: SearchFilterOptions,+): Record<string, unknown> | null {+ return buildSearchTokenFilter(USER_SEARCH_TOKEN_FIELDS, query, options);+}++export function buildGroupSearchFilter(query: string): Record<string, unknown> | null {+ return buildSearchTokenFilter(GROUP_SEARCH_TOKEN_FIELDS, query);+}