Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧬 fix: Skip Re-Embedding Legacy File Search Files After Upgrade (#17206)

Marco Beretta8fab21d8820d
diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js
index 2ec7bf5a11..4b279a7f8a 100644
--- a/api/server/controllers/agents/client.js
+++ b/api/server/controllers/agents/client.js
@@ -209,6 +209,7 @@ const {
resolvePersistableCodeEnvironmentDecision,
getChatProjectContextKey,
isTemporaryTurn,
+ hasEntityEmbeddings,
} = require('@librechat/api');
const {
Run,
@@ -3423,6 +3424,8 @@ class AgentClient extends BaseClient {
{
getProjectFiles: db.getProjectFiles,
getFiles: db.getFiles,
+ adoptLegacySearchEmbeddings: db.adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings,
getUserKey: db.getUserKey,
getConvoFiles: db.getConvoFiles,
getAccessibleMcpServerNames,
diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js
index e83e337afe..35fb0495c5 100644
--- a/api/server/controllers/agents/openai.js
+++ b/api/server/controllers/agents/openai.js
@@ -77,6 +77,7 @@ const {
createTerminalRunErrorObserver,
isTemporaryTurn,
getModelEndUsage,
+ hasEntityEmbeddings,
} = require('@librechat/api');
const {
buildSummarizationHandlers,
@@ -550,6 +551,8 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
getUserCodeFiles: db.getUserCodeFiles,
getDeferredProvisionFiles: db.getDeferredProvisionFiles,
checkSessionsAlive,
+ adoptLegacySearchEmbeddings: db.adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings,
loadCodeApiKey,
getToolFilesByIds: db.getToolFilesByIds,
getCodeGeneratedFiles: db.getCodeGeneratedFiles,
diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js
index 5bcbfe0868..512a28e890 100644
--- a/api/server/controllers/agents/responses.js
+++ b/api/server/controllers/agents/responses.js
@@ -93,6 +93,7 @@ const {
getConversationWriteContext,
isTemporaryTurn,
getModelEndUsage,
+ hasEntityEmbeddings,
} = require('@librechat/api');
const {
createResponsesToolEndCallback,
@@ -805,6 +806,8 @@ const executeResponse = async (envelope, { req, res }) => {
getUserCodeFiles: db.getUserCodeFiles,
getDeferredProvisionFiles: db.getDeferredProvisionFiles,
checkSessionsAlive,
+ adoptLegacySearchEmbeddings: db.adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings,
loadCodeApiKey,
getToolFilesByIds: db.getToolFilesByIds,
getCodeGeneratedFiles: db.getCodeGeneratedFiles,
diff --git a/api/server/services/Endpoints/agents/addedConvo.js b/api/server/services/Endpoints/agents/addedConvo.js
index e598fe46c4..93caeec43a 100644
--- a/api/server/services/Endpoints/agents/addedConvo.js
+++ b/api/server/services/Endpoints/agents/addedConvo.js
@@ -6,6 +6,7 @@ const {
resolveAgentScopedSkillIds,
resolveModelSpecSkillIds,
loadAddedAgent: loadAddedAgentFn,
+ hasEntityEmbeddings,
} = require('@librechat/api');
const { isEphemeralAgentId } = require('librechat-data-provider');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
@@ -215,6 +216,8 @@ const processAddedConvo = async ({
{
getProjectFiles: db.getProjectFiles,
getFiles: db.getFiles,
+ adoptLegacySearchEmbeddings: db.adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings,
getUserKey: db.getUserKey,
getMessages: db.getMessages,
getConvoFiles: db.getConvoFiles,
diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js
index b82704399c..976f6224b5 100644
--- a/api/server/services/Endpoints/agents/initialize.js
+++ b/api/server/services/Endpoints/agents/initialize.js
@@ -55,6 +55,7 @@ const {
getScheduleMCPExecution,
getMCPRequestContext,
createSubagentCodeRouting,
+ hasEntityEmbeddings,
} = require('@librechat/api');
const {
ResourceType,
@@ -821,6 +822,8 @@ const initializeClientWithProvider = async ({
provisionToCodeEnv,
provisionToVectorDB,
checkSessionsAlive,
+ adoptLegacySearchEmbeddings: db.adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings,
loadCodeApiKey,
updateFile: db.updateFile,
getRoleByName: db.getRoleByName,
@@ -916,6 +919,8 @@ const initializeClientWithProvider = async ({
provisionToCodeEnv,
provisionToVectorDB,
checkSessionsAlive,
+ adoptLegacySearchEmbeddings: db.adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings,
loadCodeApiKey,
updateFile: db.updateFile,
getRoleByName: db.getRoleByName,
@@ -1459,6 +1464,8 @@ const initializeClientWithProvider = async ({
provisionToCodeEnv,
provisionToVectorDB,
checkSessionsAlive,
+ adoptLegacySearchEmbeddings: db.adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings,
loadCodeApiKey,
updateFile: db.updateFile,
getRoleByName: db.getRoleByName,
@@ -1989,6 +1996,8 @@ const initializeClientWithProvider = async ({
updateFile: db.updateFile,
updateCodeEnvRef: db.updateFileCodeEnvRef,
addEmbeddedEntity: db.addFileEmbeddedEntity,
+ hasEntityEmbeddings,
+ adoptLegacySearchEmbeddings: db.adoptLegacySearchEmbeddings,
},
fileMethods: {
claimRunArtifactFile: db.claimRunArtifactFile,
diff --git a/e2e/setup/fake-rag-server.js b/e2e/setup/fake-rag-server.js
index 20469df2e0..4c7a0abbff 100644
--- a/e2e/setup/fake-rag-server.js
+++ b/e2e/setup/fake-rag-server.js
@@ -7,26 +7,37 @@
* implements just enough of the contract to exercise the vector-DB provisioning
* path end to end (real backend + DB) without a live embedding service:
*
- * - `POST /embed` — embed an uploaded file (`uploadVectors`); returns
- * `known_type: true` so the file is marked `embedded`.
- * - `POST /query` — semantic query from the `file_search` tool; returns an
- * empty result set (valid, handled gracefully).
- * - `DELETE /documents` — delete embeddings (`deleteVectors`).
+ * - `POST /embed`: embed an uploaded file (`uploadVectors`); returns
+ * `known_type: true` so the file is marked `embedded`.
+ * - `POST /query`: semantic query from the `file_search` tool; returns an
+ * empty result set (valid, handled gracefully). A query for `PROBE_K` chunks
+ * is priming's presence check, recorded under `probes` and answered from the
+ * stored embeds the way RAG authorizes chunks: a hit only when the file was
+ * embedded under the queried `entity_id`.
+ * - `DELETE /documents`: delete embeddings (`deleteVectors`).
*
* Every embed is recorded and surfaced at `GET /__debug/embedded` so specs can
* assert a file's bytes actually reached the RAG env, independent of the DB write.
+ * `/__debug/reset` clears those recordings only; the stored embeds behind presence
+ * checks persist like real vectors until `DELETE /documents` removes them.
*/
const http = require('http');
const busboy = require('busboy');
const PORT = parseInt(process.env.E2E_RAG_API_PORT || '8791', 10);
+/** Mirrors ENTITY_EMBEDDINGS_PROBE_K in packages/api/src/files/rag.ts; file_search asks for 5. */
+const PROBE_K = 10;
const HOST = '127.0.0.1';
/** @type {Array<{ file_id: string; filename: string; entity_id: string; bytes: number; auth: string }>} */
const embedded = [];
/** @type {Array<{ file_id: string; query: string }>} */
const queries = [];
+/** @type {Array<{ file_id: string; entity_id: string; present: boolean }>} */
+const probes = [];
+/** @type {Array<{ file_id: string; entity_id: string }>} */
+const stored = [];
/** @type {string[]} */
const deleted = [];
@@ -87,19 +98,49 @@ async function handleEmbed(req, res) {
bytes: files[0]?.bytes ?? 0,
auth: req.headers['authorization'] || '',
});
+ stored.push({ file_id: fields.file_id || '', entity_id: fields.entity_id || '' });
sendJson(res, 200, { status: true, known_type: true });
}
async function handleQuery(req, res) {
const body = await readJson(req);
- queries.push({ file_id: body.file_id || '', query: body.query || '' });
- sendJson(res, 200, []);
+ const entityId = body.entity_id || '';
+ if (body.k !== PROBE_K) {
+ queries.push({ file_id: body.file_id || '', query: body.query || '' });
+ sendJson(res, 200, []);
+ return;
+ }
+ const present = stored.some(
+ (entry) => entry.file_id === body.file_id && entry.entity_id === entityId,
+ );
+ probes.push({ file_id: body.file_id || '', entity_id: entityId, present });
+ sendJson(
+ res,
+ 200,
+ present
+ ? [
+ [
+ {
+ page_content: '',
+ metadata: { source: '', file_id: body.file_id, user_id: entityId },
+ },
+ 0,
+ ],
+ ]
+ : [],
+ );
}
async function handleDeleteDocuments(req, res) {
const body = await readJson(req);
if (Array.isArray(body)) {
- deleted.push(...body.map(String));
+ const ids = body.map(String);
+ deleted.push(...ids);
+ for (let i = stored.length - 1; i >= 0; i--) {
+ if (ids.includes(stored[i].file_id)) {
+ stored.splice(i, 1);
+ }
+ }
}
sendJson(res, 200, { status: true });
}
@@ -116,13 +157,14 @@ const server = http.createServer((req, res) => {
}
if (pathname === '/__debug/embedded' && req.method === 'GET') {
- sendJson(res, 200, { embedded, queries, deleted });
+ sendJson(res, 200, { embedded, queries, probes, deleted });
return;
}
if (pathname === '/__debug/reset' && req.method === 'POST') {
embedded.length = 0;
queries.length = 0;
+ probes.length = 0;
deleted.length = 0;
sendJson(res, 200, { ok: true });
return;
diff --git a/e2e/specs/mock/scenarios/legacy-search-embedding.spec.ts b/e2e/specs/mock/scenarios/legacy-search-embedding.spec.ts
new file mode 100644
index 0000000000..b343eaa70a
--- /dev/null
+++ b/e2e/specs/mock/scenarios/legacy-search-embedding.spec.ts
@@ -0,0 +1,208 @@
+import { randomUUID } from 'node:crypto';
+import { expect, test } from '@playwright/test';
+import type { UploadedFile } from '../helpers';
+import type { AgentDetail } from '../agents.helpers';
+import { cleanupAgent } from '../agents.helpers';
+import { withMongo } from '../db';
+import {
+ MOCK_ENDPOINTS,
+ NEW_CHAT_PATH,
+ uniqueName,
+ requestJson,
+ getRagQueries,
+ getAccessToken,
+ getRagEmbedded,
+ resetProvisioning,
+ sendMessageAndWaitForCompletion,
+} from '../helpers';
+
+/**
+ * Files uploaded for File Search before v0.8.8 carry `embedded: true` but no
+ * `metadata.embeddedEntities`. They must stay searchable without being embedded a second
+ * time, which would store a duplicate copy under the same file_id (#16843).
+ */
+test.describe('legacy File Search embeddings', () => {
+ test('an agent file embedded before namespace tracking is searched without re-embedding @scenario:legacy-search-file-not-re-embedded', async ({
+ page,
+ }) => {
+ test.setTimeout(180000);
+ let agentId: string | undefined;
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+
+ try {
+ const token = await getAccessToken(page);
+ const agent = await requestJson<AgentDetail>(page, {
+ path: '/api/agents',
+ token,
+ method: 'POST',
+ body: {
+ name: uniqueName('legacy-search-agent'),
+ description: 'Legacy File Search embedding fixture.',
+ instructions: 'Search the attached files.',
+ provider: MOCK_ENDPOINTS[0].label,
+ model: MOCK_ENDPOINTS[0].model,
+ tools: ['file_search'],
+ },
+ });
+ agentId = agent.id;
+
+ const upload = await page.request.post('/api/files', {
+ headers: { Authorization: `Bearer ${token}` },
+ multipart: {
+ endpoint: MOCK_ENDPOINTS[0].label,
+ endpointType: 'custom',
+ agent_id: agentId,
+ tool_resource: 'file_search',
+ file_id: randomUUID(),
+ file: {
+ name: `${uniqueName('legacy-search')}.txt`,
+ mimeType: 'text/plain',
+ buffer: Buffer.from('Legacy search fixture text.'),
+ },
+ },
+ });
+ expect(upload.ok(), await upload.text()).toBeTruthy();
+ /* The server assigns the stored id; the fake RAG records it rather than a name. */
+ const fileId = ((await upload.json()) as UploadedFile).file_id as string;
+ expect(fileId, 'upload response should include a file_id').toBeTruthy();
+ await expect
+ .poll(async () => (await getRagEmbedded(page)).map((e) => e.file_id), { timeout: 15000 })
+ .toContain(fileId);
+
+ /* Reshape the record into what v0.8.7 wrote: embedded, with no namespace list. */
+ const reshaped = await withMongo((db) =>
+ db
+ .collection('files')
+ .updateOne(
+ { file_id: fileId, embedded: true },
+ { $unset: { 'metadata.embeddedEntities': '' } },
+ ),
+ );
+ expect(reshaped.matchedCount).toBe(1);
+ await resetProvisioning(page);
+
+ await page.goto(`${NEW_CHAT_PATH}?agent_id=${encodeURIComponent(agentId)}`, {
+ timeout: 10000,
+ });
+ await sendMessageAndWaitForCompletion(page, `E2E_FILE_SEARCH:${uniqueName('legacy')}`);
+
+ await expect
+ .poll(async () => (await getRagQueries(page)).filter((q) => q.file_id === fileId).length, {
+ timeout: 30000,
+ })
+ .toBeGreaterThan(0);
+ expect(
+ (await getRagEmbedded(page)).filter((e) => e.file_id === fileId),
+ 'a legacy embedded File Search file must not be embedded again',
+ ).toHaveLength(0);
+ } finally {
+ await cleanupAgent(page, agentId);
+ }
+ });
+
+ test('a legacy file another agent also holds is embedded for that agent only @scenario:legacy-search-file-shared-re-embedded', async ({
+ page,
+ }) => {
+ test.setTimeout(180000);
+ let sourceId: string | undefined;
+ let inheritorId: string | undefined;
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+
+ try {
+ const token = await getAccessToken(page);
+ const createSearchAgent = (label: string) =>
+ requestJson<AgentDetail>(page, {
+ path: '/api/agents',
+ token,
+ method: 'POST',
+ body: {
+ name: uniqueName(label),
+ description: 'Legacy File Search embedding fixture.',
+ instructions: 'Search the attached files.',
+ provider: MOCK_ENDPOINTS[0].label,
+ model: MOCK_ENDPOINTS[0].model,
+ tools: ['file_search'],
+ },
+ });
+ sourceId = (await createSearchAgent('legacy-source-agent')).id;
+ inheritorId = (await createSearchAgent('legacy-inheritor-agent')).id;
+
+ const upload = await page.request.post('/api/files', {
+ headers: { Authorization: `Bearer ${token}` },
+ multipart: {
+ endpoint: MOCK_ENDPOINTS[0].label,
+ endpointType: 'custom',
+ agent_id: sourceId,
+ tool_resource: 'file_search',
+ file_id: randomUUID(),
+ file: {
+ name: `${uniqueName('legacy-shared')}.txt`,
+ mimeType: 'text/plain',
+ buffer: Buffer.from('Legacy shared search fixture text.'),
+ },
+ },
+ });
+ expect(upload.ok(), await upload.text()).toBeTruthy();
+ /* The server assigns the stored id; the fake RAG records it rather than a name. */
+ const fileId = ((await upload.json()) as UploadedFile).file_id as string;
+ expect(fileId, 'upload response should include a file_id').toBeTruthy();
+ await expect
+ .poll(async () => (await getRagEmbedded(page)).map((e) => e.file_id), { timeout: 15000 })
+ .toContain(fileId);
+
+ /* The owner of a file may attach it to another of their agents. */
+ await requestJson<AgentDetail>(page, {
+ path: `/api/agents/${encodeURIComponent(inheritorId)}`,
+ token,
+ method: 'PATCH',
+ body: { tool_resources: { file_search: { file_ids: [fileId] } } },
+ });
+ const reshaped = await withMongo((db) =>
+ db
+ .collection('files')
+ .updateOne(
+ { file_id: fileId, embedded: true },
+ { $unset: { 'metadata.embeddedEntities': '' } },
+ ),
+ );
+ expect(reshaped.matchedCount).toBe(1);
+ await resetProvisioning(page);
+
+ await page.goto(`${NEW_CHAT_PATH}?agent_id=${encodeURIComponent(inheritorId)}`, {
+ timeout: 10000,
+ });
+ await sendMessageAndWaitForCompletion(page, `E2E_FILE_SEARCH:${uniqueName('shared')}`);
+
+ /* Its vectors are in the source agent's namespace, so the searching agent gets its
+ * own embedding rather than querying a namespace that holds none. */
+ await expect
+ .poll(async () => (await getRagEmbedded(page)).filter((e) => e.file_id === fileId).length, {
+ timeout: 30000,
+ })
+ .toBe(1);
+ await expect
+ .poll(async () => (await getRagQueries(page)).filter((q) => q.file_id === fileId).length)
+ .toBeGreaterThan(0);
+
+ /* The record now lists only the inheritor, yet the source agent's own vectors are still
+ * there, so its next search must not embed a second copy. */
+ await resetProvisioning(page);
+ await page.goto(`${NEW_CHAT_PATH}?agent_id=${encodeURIComponent(sourceId)}`, {
+ timeout: 10000,
+ });
+ await sendMessageAndWaitForCompletion(page, `E2E_FILE_SEARCH:${uniqueName('source')}`);
+ await expect
+ .poll(async () => (await getRagQueries(page)).filter((q) => q.file_id === fileId).length, {
+ timeout: 30000,
+ })
+ .toBeGreaterThan(0);
+ expect(
+ (await getRagEmbedded(page)).filter((e) => e.file_id === fileId),
+ 'the agent that embedded the file must not embed it again',
+ ).toHaveLength(0);
+ } finally {
+ await cleanupAgent(page, inheritorId);
+ await cleanupAgent(page, sourceId);
+ }
+ });
+});
diff --git a/packages/api/src/agents/files/host.spec.ts b/packages/api/src/agents/files/host.spec.ts
index 46c1d631af..51cea7b292 100644
--- a/packages/api/src/agents/files/host.spec.ts
+++ b/packages/api/src/agents/files/host.spec.ts
@@ -68,7 +68,14 @@ function deferred<T>() {
return { promise, resolve, reject };
}
-function harness(options: { inputs?: TFile[]; setup?: TFile[] } = {}) {
+function harness(
+ options: {
+ inputs?: TFile[];
+ setup?: TFile[];
+ hasEntityEmbeddings?: jest.Mock;
+ adoptLegacySearchEmbeddings?: jest.Mock;
+ } = {},
+) {
const input = options.inputs ?? [file('input')];
const setup = options.setup ?? [file('worker-setup', { context: FileContext.agents })];
const parsed = configSchema.parse({
@@ -198,6 +205,8 @@ function harness(options: { inputs?: TFile[]; setup?: TFile[] } = {}) {
updateCodeEnvRef,
updateFile: jest.fn(async () => undefined),
addEmbeddedEntity,
+ hasEntityEmbeddings: options.hasEntityEmbeddings,
+ adoptLegacySearchEmbeddings: options.adoptLegacySearchEmbeddings,
},
});
const deps = spy.mock.calls[spy.mock.calls.length - 1][0];
@@ -387,6 +396,27 @@ describe('run file execution host', () => {
).toEqual(['historical']);
});
+ it('confirms an unrecorded setup embedding instead of embedding it again', async () => {
+ /* A worker's File Search setup file embedded before namespaces were tracked already has
+ * vectors in the worker's namespace; a shared-file run must not store a second copy. */
+ const hasEntityEmbeddings = jest.fn().mockResolvedValue(true);
+ const adoptLegacySearchEmbeddings = jest.fn().mockResolvedValue(['worker-setup']);
+ const h = harness({
+ setup: [file('worker-setup', { context: FileContext.agents, embedded: true })],
+ hasEntityEmbeddings,
+ adoptLegacySearchEmbeddings,
+ });
+ await h.prepare();
+ await h.host.provision(['file_search'], 'worker', h.signal, identity());
+
+ expect(hasEntityEmbeddings).toHaveBeenCalledWith(
+ expect.objectContaining({ file_id: 'worker-setup', entity_id: 'worker' }),
+ );
+ expect(h.provisionToVectorDB.mock.calls.map(([params]) => params.file.file_id)).not.toContain(
+ 'worker-setup',
+ );
+ });
+
it('retains only admitted advertised paths when preparing isolated child queues', async () => {
const current = file('input', { filename: 'data.csv' });
const h = harness({ inputs: [current] });
diff --git a/packages/api/src/agents/files/host.ts b/packages/api/src/agents/files/host.ts
index b1b140e254..bcd07a4eb1 100644
--- a/packages/api/src/agents/files/host.ts
+++ b/packages/api/src/agents/files/host.ts
@@ -1,9 +1,13 @@
import { EToolResources, getCodeEnvRefs } from 'librechat-data-provider';
import type { CodeEnvFile, SubagentExecutionContext } from '@librechat/agents';
import type { Agent, TFile } from 'librechat-data-provider';
+import type {
+ THasEntityEmbeddings,
+ TFilterFilesByAgentAccess,
+ TAdoptLegacySearchEmbeddings,
+} from '~/agents/resources';
import type { ProvisionCallbackDeps, ProvisionToolContext } from '~/files/provision/callback';
import type { ToolEndCallback, ToolEndCallbackMetadata } from '~/agents/handlers';
-import type { TFilterFilesByAgentAccess } from '~/agents/resources';
import type { RunFileSession, RunFileSessionDeps } from './session';
import type { CodeExecutionContext } from '~/agents/execution';
import type { ServerRequest } from '~/types';
@@ -108,7 +112,11 @@ export function createRunFileHost<TContext extends RunFileToolContext>({
| 'updateFile'
| 'updateCodeEnvRef'
| 'addEmbeddedEntity'
- >;
+ > & {
+ /** Lets setup files whose namespace is unrecorded be confirmed instead of embedded again. */
+ hasEntityEmbeddings?: THasEntityEmbeddings;
+ adoptLegacySearchEmbeddings?: TAdoptLegacySearchEmbeddings;
+ };
}): RunFileHost<TContext> {
const user = req.user;
if (user == null) throw new Error('Run file sharing requires an authenticated user.');
@@ -342,6 +350,8 @@ export function createRunFileHost<TContext extends RunFileToolContext>({
attachments: Promise.resolve(currentFiles),
requestFileSet: new Set(currentFiles.map((file) => file.file_id)),
tool_resources: source.agent.tool_resources,
+ hasEntityEmbeddings: provisioning.hasEntityEmbeddings,
+ adoptLegacySearchEmbeddings: provisioning.adoptLegacySearchEmbeddings,
enabledToolResources: resolveResendToolResources({
tools: source.agent.tools ?? [],
codeEnvAvailable: source.codeEnvAvailable === true,
diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts
index c37937cb15..1bed769edd 100644
--- a/packages/api/src/agents/initialize.ts
+++ b/packages/api/src/agents/initialize.ts
@@ -37,6 +37,8 @@ import type {
TFilterFilesByAgentAccess,
TProvisionToCodeEnv,
TProvisionToVectorDB,
+ TAdoptLegacySearchEmbeddings,
+ THasEntityEmbeddings,
TCheckSessionsAlive,
TLoadCodeApiKey,
} from './resources';
@@ -1142,6 +1144,10 @@ export interface InitializeAgentDbMethods extends EndpointDbMethods {
provisionToVectorDB?: TProvisionToVectorDB;
/** Optional: batch-check code env file liveness */
checkSessionsAlive?: TCheckSessionsAlive;
+ /** Optional: record the agent's namespace on legacy File Search embeddings confirmed there */
+ adoptLegacySearchEmbeddings?: TAdoptLegacySearchEmbeddings;
+ /** Optional: ask the vector store whether a file's chunks are in an entity's namespace */
+ hasEntityEmbeddings?: THasEntityEmbeddings;
/** Optional: load CODE_API_KEY once per request */
loadCodeApiKey?: TLoadCodeApiKey;
/** Optional: persist file metadata updates after provisioning */
@@ -2047,6 +2053,8 @@ export async function initializeAgent(
enabledToolResources: toolResourceSet,
checkSessionsAlive: db.checkSessionsAlive,
loadCodeApiKey: db.loadCodeApiKey,
+ adoptLegacySearchEmbeddings: db.adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings: db.hasEntityEmbeddings,
provisionCandidates: deferredProvisionFiles as unknown as TFile[],
codeRouteKey: codeExecutionContext.executionRouteKey ?? codeExecutionContext.executionProfile,
codeBaseUrl: codeExecutionContext.baseUrl,
diff --git a/packages/api/src/agents/resources.test.ts b/packages/api/src/agents/resources.test.ts
index 9e3530be7d..46f43f2b63 100644
--- a/packages/api/src/agents/resources.test.ts
+++ b/packages/api/src/agents/resources.test.ts
@@ -2667,6 +2667,214 @@ describe('primeResources', () => {
]);
});
+ describe('legacy File Search files held by the agent', () => {
+ /* Before namespaces were tracked, a file joined an agent's file_search resources only
+ * by being embedded for it. Embedding it again stores a second copy under the same
+ * file_id, and every search returns each passage twice. */
+ const legacySearchFile = (): TFile =>
+ ({
+ user: 'user1',
+ file_id: 'legacy-search',
+ filename: 'handbook.pdf',
+ filepath: '/uploads/handbook.pdf',
+ object: 'file',
+ type: 'application/pdf',
+ bytes: 2048,
+ embedded: true,
+ usage: 0,
+ source: FileSources.local,
+ context: FileContext.agents,
+ }) as TFile;
+
+ const primeLegacy = (
+ hasEntityEmbeddings: jest.Mock,
+ adoptLegacySearchEmbeddings: jest.Mock,
+ ) => {
+ mockGetFiles.mockResolvedValue([legacySearchFile()]);
+ return primeResources({
+ req: mockReq,
+ appConfig: mockAppConfig,
+ getFiles: mockGetFiles,
+ filterFiles: mockFilterFiles,
+ tool_resources: { [EToolResources.file_search]: { file_ids: ['legacy-search'] } },
+ attachments: undefined,
+ requestFileSet,
+ agentId: 'agent1',
+ enabledToolResources: new Set([EToolResources.file_search]),
+ hasEntityEmbeddings,
+ adoptLegacySearchEmbeddings,
+ });
+ };
+
+ it("does not re-embed a file whose vectors are in this agent's namespace", async () => {
+ const probe = jest.fn().mockResolvedValue(true);
+ const adopt = jest.fn().mockResolvedValue(['legacy-search']);
+
+ const result = await primeLegacy(probe, adopt);
+
+ expect(probe).toHaveBeenCalledWith({
+ userId: 'user1',
+ file_id: 'legacy-search',
+ entity_id: 'agent1',
+ query: 'handbook.pdf',
+ timeoutMs: 5000,
+ });
+ expect(adopt).toHaveBeenCalledWith({
+ files: [{ file_id: 'legacy-search', user: 'user1' }],
+ agentId: 'agent1',
+ });
+ expect(result.provisionState).toBeUndefined();
+ expect(result.tool_resources?.[EToolResources.file_search]?.file_ids).toEqual([
+ 'legacy-search',
+ ]);
+ });
+
+ it('bounds the check by the configured timeout', async () => {
+ const probe = jest.fn().mockResolvedValue(true);
+ mockGetFiles.mockResolvedValue([legacySearchFile()]);
+
+ await primeResources({
+ req: mockReq,
+ appConfig: {
+ ...mockAppConfig,
+ fileConfig: { ...mockAppConfig.fileConfig, legacySearchProbeTimeoutMs: 1234 },
+ } as AppConfig,
+ getFiles: mockGetFiles,
+ filterFiles: mockFilterFiles,
+ tool_resources: { [EToolResources.file_search]: { file_ids: ['legacy-search'] } },
+ attachments: undefined,
+ requestFileSet,
+ agentId: 'agent1',
+ enabledToolResources: new Set([EToolResources.file_search]),
+ hasEntityEmbeddings: probe,
+ adoptLegacySearchEmbeddings: jest.fn().mockResolvedValue(['legacy-search']),
+ });
+
+ expect(probe).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 1234 }));
+ });
+
+ it('checks a file another agent recorded first before embedding it here', async () => {
+ /* An inheriting agent embedded and recorded it first; this agent's own older vectors
+ * went unrecorded, so embedding again would duplicate them. */
+ const probe = jest.fn().mockResolvedValue(true);
+ const adopt = jest.fn().mockResolvedValue(['legacy-search']);
+ mockGetFiles.mockResolvedValue([
+ { ...legacySearchFile(), metadata: { embeddedEntities: ['agent2'] } } as TFile,
+ ]);
+
+ const result = await primeResources({
+ req: mockReq,
+ appConfig: mockAppConfig,
+ getFiles: mockGetFiles,
+ filterFiles: mockFilterFiles,
+ tool_resources: { [EToolResources.file_search]: { file_ids: ['legacy-search'] } },
+ attachments: undefined,
+ requestFileSet,
+ agentId: 'agent1',
+ enabledToolResources: new Set([EToolResources.file_search]),
+ hasEntityEmbeddings: probe,
+ adoptLegacySearchEmbeddings: adopt,
+ });
+
+ expect(probe).toHaveBeenCalledWith(expect.objectContaining({ entity_id: 'agent1' }));
+ expect(result.provisionState).toBeUndefined();
+ });
+
+ it("embeds a file whose vectors are not in this agent's namespace", async () => {
+ /* Another agent embedded it, so this agent's namespace holds none of its vectors and
+ * search would return nothing. */
+ const probe = jest.fn().mockResolvedValue(false);
+ const adopt = jest.fn();
+
+ const result = await primeLegacy(probe, adopt);
+
+ expect(adopt).not.toHaveBeenCalled();
+ expect(result.provisionState?.vectorDBFiles.map((f) => f.file_id)).toEqual([
+ 'legacy-search',
+ ]);
+ });
+
+ it('logs a failed check without the vector store response body', async () => {
+ const error = Object.assign(new Error('Request failed with status code 500'), {
+ response: { status: 500, data: { detail: 'upstream secret text' } },
+ });
+ const probe = jest.fn().mockRejectedValue(error);
+
+ await primeLegacy(probe, jest.fn());
+
+ expect(logger.error).toHaveBeenCalled();
+ expect(JSON.stringify((logger.error as jest.Mock).mock.calls)).not.toContain(
+ 'upstream secret text',
+ );
+ });
+
+ it('keeps the file queued when the vector store cannot be asked', async () => {
+ const probe = jest.fn().mockRejectedValue(new Error('connect ECONNREFUSED'));
+ const adopt = jest.fn();
+
+ const result = await primeLegacy(probe, adopt);
+
+ expect(adopt).not.toHaveBeenCalled();
+ expect(result.provisionState?.vectorDBFiles.map((f) => f.file_id)).toEqual([
+ 'legacy-search',
+ ]);
+ });
+
+ it('uses confirmed vectors even when recording the namespace fails', async () => {
+ /* The vectors are there; embedding again would store the duplicate copy. */
+ const probe = jest.fn().mockResolvedValue(true);
+ const adopt = jest.fn().mockRejectedValue(new Error('database unavailable'));
+
+ const result = await primeLegacy(probe, adopt);
+
+ expect(result.provisionState).toBeUndefined();
+ });
+
+ it('starts the legacy check before the code liveness probe settles', async () => {
+ const probe = jest.fn().mockResolvedValue(true);
+ const adopt = jest.fn().mockResolvedValue(['legacy-search']);
+ let checkStartedFirst = false;
+ const checkSessionsAlive = jest.fn(async () => {
+ checkStartedFirst = probe.mock.calls.length === 1;
+ return new Set(['legacy-search']);
+ });
+ mockGetFiles.mockResolvedValue([
+ {
+ ...legacySearchFile(),
+ metadata: {
+ codeEnvRef: {
+ kind: 'agent',
+ id: 'agent1',
+ storage_session_id: 'sess',
+ file_id: 'fid',
+ },
+ },
+ } as TFile,
+ ]);
+
+ await primeResources({
+ req: mockReq,
+ appConfig: mockAppConfig,
+ getFiles: mockGetFiles,
+ filterFiles: mockFilterFiles,
+ tool_resources: {
+ [EToolResources.file_search]: { file_ids: ['legacy-search'] },
+ [EToolResources.execute_code]: { file_ids: ['legacy-search'] },
+ },
+ attachments: undefined,
+ requestFileSet,
+ agentId: 'agent1',
+ enabledToolResources: new Set([EToolResources.file_search, EToolResources.execute_code]),
+ checkSessionsAlive,
+ hasEntityEmbeddings: probe,
+ adoptLegacySearchEmbeddings: adopt,
+ });
+
+ expect(checkSessionsAlive).toHaveBeenCalled();
+ expect(checkStartedFirst).toBe(true);
+ });
+ });
+
it('leaves a file alone whose destination came from the legacy chooser', async () => {
/* The endpoint deciding this turn need not be the one the file was uploaded under,
* so the request-level legacy check cannot answer for it. Its missing references
diff --git a/packages/api/src/agents/resources.ts b/packages/api/src/agents/resources.ts
index 743b2e6a04..0a5d6fadc4 100644
--- a/packages/api/src/agents/resources.ts
+++ b/packages/api/src/agents/resources.ts
@@ -21,6 +21,7 @@ import type { FilterQuery, QueryOptions, ProjectionType } from 'mongoose';
import type { CodeExecutionContext } from './execution';
import type { ServerRequest } from '~/types';
+import { getSafeErrorMetadata } from '~/utils/errors';
import { TOOL_RESOURCE_KEYS } from './orphans';
/** Removes runtime-only file records before persisted Agent resources enter tool initialization. */
@@ -87,6 +88,31 @@ export type TCheckSessionsAlive = (params: {
/** Loads CODE_API_KEY for a user. Call once per request. */
export type TLoadCodeApiKey = (userId: string) => Promise<string>;
+/**
+ * Records the agent's namespace on legacy embedded files already confirmed to have vectors
+ * there, and returns the ids it recorded.
+ */
+export type TAdoptLegacySearchEmbeddings = (params: {
+ files: Array<{ file_id: string; user: string; tenantId?: string }>;
+ agentId: string;
+}) => Promise<string[]>;
+
+/**
+ * Whether the vector store holds chunks of a file in an entity's namespace. Records from
+ * before namespaces were tracked say only that a file was embedded somewhere, so this is
+ * the evidence that decides whether embedding it again would store a second copy.
+ */
+export type THasEntityEmbeddings = (params: {
+ userId: string;
+ file_id: string;
+ entity_id: string;
+ query: string;
+ timeoutMs?: number;
+}) => Promise<boolean>;
+
+/** How long priming waits on a legacy File Search check before embedding the file anyway. */
+const DEFAULT_LEGACY_SEARCH_PROBE_TIMEOUT_MS = 5000;
+
/** State computed during primeResources for lazy provisioning at tool invocation time */
export type ProvisionState = {
/** Files that need uploading to the code execution environment */
@@ -201,7 +227,8 @@ export const isAgentScopedFile = (file: Pick<TFile, 'context'>): boolean =>
* setup file attached to a second agent is embedded under the first agent's entity, so
* trusting the flag registers it for a namespace that holds none of its vectors and the
* search returns nothing. Records embedded before namespaces were tracked carry no entity
- * list and are re-embedded once per namespace, which repairs the record as it goes.
+ * list and are re-embedded once per namespace, which repairs the record as it goes, unless
+ * the vector store confirms the vectors are already there (see isUnrecordedSearchFile).
*/
const isEmbeddedForNamespace = (file: TFile, namespaceId?: string): boolean => {
if (!isAgentScopedFile(file)) {
@@ -210,6 +237,23 @@ const isEmbeddedForNamespace = (file: TFile, namespaceId?: string): boolean => {
return namespaceId != null && file.metadata?.embeddedEntities?.includes(namespaceId) === true;
};
+/**
+ * Whether an embedded File Search file of the active agent records no embedding for its
+ * namespace. Records from before namespaces were tracked can hold vectors there regardless:
+ * a file another agent embedded first lists only that agent, while this agent's own older
+ * vectors went unrecorded. Only the vector store can say; see THasEntityEmbeddings.
+ */
+const isUnrecordedSearchFile = (
+ file: TFile,
+ agentId: string,
+ membership?: ReadonlyMap<EToolResources, ReadonlySet<string>>,
+): boolean =>
+ isAgentScopedFile(file) &&
+ file.embedded === true &&
+ !isEmbeddedForNamespace(file, agentId) &&
+ file.file_id != null &&
+ membership?.get(EToolResources.file_search)?.has(file.file_id) === true;
+
/** The scope a code upload would be filed under this turn, mirroring what
* provisionToCodeEnv records: an agent's own resource file belongs to the agent, and
* everything else to the requesting user. */
@@ -409,6 +453,9 @@ const computeProvisionState = async ({
codeBridgeWorkerId,
agentScopedFileIds,
persistedResourceMembership,
+ adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings,
+ legacySearchProbeTimeoutMs = DEFAULT_LEGACY_SEARCH_PROBE_TIMEOUT_MS,
staleSafeWindowMs,
}: {
req?: ServerRequest;
@@ -429,6 +476,9 @@ const computeProvisionState = async ({
loadCodeApiKey?: TLoadCodeApiKey;
/** Which resource each persisted file is filed under, before categorization mutates it. */
persistedResourceMembership?: ReadonlyMap<EToolResources, ReadonlySet<string>>;
+ adoptLegacySearchEmbeddings?: TAdoptLegacySearchEmbeddings;
+ hasEntityEmbeddings?: THasEntityEmbeddings;
+ legacySearchProbeTimeoutMs?: number;
staleSafeWindowMs?: number;
}): Promise<ProvisionState | undefined> => {
if (!enabledToolResources || enabledToolResources.size === 0 || attachments.length === 0) {
@@ -482,6 +532,72 @@ const computeProvisionState = async ({
return undefined;
}
+ /* Started before the liveness probe, with which it shares nothing. Adoption only spares a
+ * re-embed, so a failure at any step leaves the files queued rather than failing priming. */
+ const legacySearchFiles =
+ needsVectorDB && agentId != null && resourcePrincipal?.id != null
+ ? provisionable.filter((file) =>
+ isUnrecordedSearchFile(file, agentId, persistedResourceMembership),
+ )
+ : [];
+ const adoptLegacy = async (): Promise<Set<string>> => {
+ if (
+ agentId == null ||
+ resourcePrincipal?.id == null ||
+ !hasEntityEmbeddings ||
+ !adoptLegacySearchEmbeddings
+ ) {
+ return new Set();
+ }
+ const present = await Promise.all(
+ legacySearchFiles.map((file) =>
+ hasEntityEmbeddings({
+ userId: resourcePrincipal.id as string,
+ file_id: file.file_id,
+ entity_id: agentId,
+ query: file.filename || file.file_id,
+ timeoutMs: legacySearchProbeTimeoutMs,
+ }).catch((error) => {
+ logger.error(
+ `[primeResources] Could not check legacy embeddings for file ${file.file_id}`,
+ getSafeErrorMetadata(error),
+ );
+ return false;
+ }),
+ ),
+ );
+ const confirmed = legacySearchFiles.filter((_file, index) => present[index]);
+ if (confirmed.length === 0) {
+ return new Set();
+ }
+ /* The vector store has confirmed the vectors, so this turn uses them whether or not the
+ * namespace is recorded; recording only spares the next turn the same check. */
+ await adoptLegacySearchEmbeddings({
+ files: confirmed.map((file) => ({
+ file_id: file.file_id,
+ user: String(file.user),
+ ...(file.tenantId ? { tenantId: file.tenantId } : {}),
+ })),
+ agentId,
+ }).catch((error) => {
+ logger.error(
+ '[primeResources] Failed to record legacy File Search namespaces',
+ getSafeErrorMetadata(error),
+ );
+ });
+ return new Set(confirmed.map((file) => file.file_id));
+ };
+ const legacyAdoption: Promise<Set<string>> =
+ legacySearchFiles.length > 0
+ ? adoptLegacy().catch((error) => {
+ logger.error(
+ '[primeResources] Failed to adopt legacy File Search embeddings',
+ getSafeErrorMetadata(error),
+ );
+ return new Set<string>();
+ })
+ : Promise.resolve(new Set<string>());
+
const activeCodeRouteKey = codeRouteKey ?? 'default';
/** Batch staleness check: identify which code env files are still alive. Only files
@@ -531,6 +647,15 @@ const computeProvisionState = async ({
});
}
+ const adopted = await legacyAdoption;
+ for (const file of provisionable) {
+ if (agentId != null && adopted.has(file.file_id)) {
+ const recorded = new Set(file.metadata?.embeddedEntities ?? []);
+ recorded.add(agentId);
+ file.metadata = { ...file.metadata, embeddedEntities: [...recorded] };
+ }
+ }
+
const scopedIds = agentScopedFileIds ?? new Set<string>();
const codeEnvFiles: TFile[] = [];
const codeEnvRecoveryNames: NonNullable<ProvisionState['codeEnvRecoveryNames']> = new Map();
@@ -657,6 +782,8 @@ export const primeResources = async ({
loadCodeApiKey,
provisionCandidates,
screenPersistentFiles,
+ adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings,
codeRouteKey,
codeBaseUrl,
codeExecutionProfile,
@@ -687,6 +814,10 @@ export const primeResources = async ({
* and a configuration or policy change since they were attached would otherwise let
* their bytes reach the model, the Code API or RAG. */
screenPersistentFiles?: (files: Array<TFile>) => Array<TFile>;
+ /** Records the agent's namespace on legacy File Search embeddings confirmed to be there. */
+ adoptLegacySearchEmbeddings?: TAdoptLegacySearchEmbeddings;
+ /** Asks the vector store whether a legacy file's chunks are in the agent's namespace. */
+ hasEntityEmbeddings?: THasEntityEmbeddings;
/** The code deployment this turn will execute on, which decides whether an existing
* sandbox reference is usable here. */
codeRouteKey?: string;
@@ -930,6 +1061,9 @@ export const primeResources = async ({
codeBridgeWorkerId,
agentScopedFileIds: persistedResourceFileIds,
persistedResourceMembership,
+ adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings,
+ legacySearchProbeTimeoutMs: appConfig?.fileConfig?.legacySearchProbeTimeoutMs,
staleSafeWindowMs: appConfig?.fileConfig?.codeEnvLivenessSafeWindowMs,
});
return {
@@ -996,6 +1130,9 @@ export const primeResources = async ({
codeBridgeWorkerId,
agentScopedFileIds: persistedResourceFileIds,
persistedResourceMembership,
+ adoptLegacySearchEmbeddings,
+ hasEntityEmbeddings,
+ legacySearchProbeTimeoutMs: appConfig?.fileConfig?.legacySearchProbeTimeoutMs,
staleSafeWindowMs: appConfig?.fileConfig?.codeEnvLivenessSafeWindowMs,
});
diff --git a/packages/api/src/files/rag.spec.ts b/packages/api/src/files/rag.spec.ts
index 4a22d4dd3e..3aab3ba239 100644
--- a/packages/api/src/files/rag.spec.ts
+++ b/packages/api/src/files/rag.spec.ts
@@ -12,6 +12,7 @@ jest.mock('~/crypto/jwt', () => ({
jest.mock('axios', () => ({
delete: jest.fn(),
+ post: jest.fn(),
interceptors: {
request: { use: jest.fn(), eject: jest.fn() },
response: { use: jest.fn(), eject: jest.fn() },
@@ -19,7 +20,7 @@ jest.mock('axios', () => ({
}));
import axios from 'axios';
-import { deleteRagFile } from './rag';
+import { deleteRagFile, hasEntityEmbeddings } from './rag';
import { logger } from '@librechat/data-schemas';
import { generateShortLivedToken } from '~/crypto/jwt';
@@ -148,3 +149,85 @@ describe('deleteRagFile', () => {
});
});
});
+
+describe('hasEntityEmbeddings', () => {
+ const originalEnv = process.env;
+ const params = { userId: 'user1', file_id: 'file1', entity_id: 'agent1', query: 'notes.pdf' };
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ process.env = { ...originalEnv, RAG_API_URL: 'http://localhost:8000' };
+ });
+
+ afterEach(() => {
+ process.env = originalEnv;
+ });
+
+ it('asks RAG for chunks of the file under the entity', async () => {
+ mockedAxios.post.mockResolvedValueOnce({
+ data: [[{ page_content: 'x', metadata: { user_id: 'agent1' } }, 0.1]],
+ });
+
+ await expect(hasEntityEmbeddings(params)).resolves.toBe(true);
+ expect(mockedAxios.post).toHaveBeenCalledWith(
+ 'http://localhost:8000/query',
+ { file_id: 'file1', query: 'notes.pdf', k: 10, entity_id: 'agent1' },
+ {
+ headers: {
+ Authorization: 'Bearer mock-jwt-token',
+ 'Content-Type': 'application/json',
+ },
+ },
+ );
+ });
+
+ it("ignores chunks RAG returns because they are the requesting user's own", async () => {
+ mockedAxios.post.mockResolvedValueOnce({
+ data: [[{ page_content: 'x', metadata: { user_id: 'user1' } }, 0.1]],
+ });
+
+ await expect(hasEntityEmbeddings(params)).resolves.toBe(false);
+ });
+
+ it("finds the entity's chunks behind a requesting user's copy of the file", async () => {
+ mockedAxios.post.mockResolvedValueOnce({
+ data: [
+ [{ page_content: 'x', metadata: { user_id: 'user1' } }, 0.1],
+ [{ page_content: 'x', metadata: { user_id: 'agent1' } }, 0.1],
+ ],
+ });
+
+ await expect(hasEntityEmbeddings(params)).resolves.toBe(true);
+ });
+
+ it('reports no embeddings when RAG returns no chunks', async () => {
+ mockedAxios.post.mockResolvedValueOnce({ data: [] });
+
+ await expect(hasEntityEmbeddings(params)).resolves.toBe(false);
+ });
+
+ it('passes the timeout to the request', async () => {
+ mockedAxios.post.mockResolvedValueOnce({ data: [] });
+
+ await hasEntityEmbeddings({ ...params, timeoutMs: 1234 });
+
+ expect(mockedAxios.post).toHaveBeenCalledWith(
+ expect.any(String),
+ expect.any(Object),
+ expect.objectContaining({ timeout: 1234 }),
+ );
+ });
+
+ it('rejects when RAG cannot be asked', async () => {
+ mockedAxios.post.mockRejectedValueOnce(new Error('connect ECONNREFUSED'));
+
+ await expect(hasEntityEmbeddings(params)).rejects.toThrow('connect ECONNREFUSED');
+ });
+
+ it('reports no embeddings without asking when RAG is not configured', async () => {
+ delete process.env.RAG_API_URL;
+
+ await expect(hasEntityEmbeddings(params)).resolves.toBe(false);
+ expect(mockedAxios.post).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/api/src/files/rag.ts b/packages/api/src/files/rag.ts
index 7876cda56d..a52f3aa6af 100644
--- a/packages/api/src/files/rag.ts
+++ b/packages/api/src/files/rag.ts
@@ -58,3 +58,53 @@ export async function deleteRagFile({ userId, file }: DeleteRagFileParams): Prom
}
}
}
+
+/**
+ * Chunks a presence check asks for. RAG ranks chunks across every owner of a file_id and,
+ * once the top one is authorized, returns all of them, so a copy of the same file embedded
+ * for the requesting user ties with the entity's copy; asking for several keeps the
+ * entity's chunks in the answer whichever copy ranks first.
+ */
+export const ENTITY_EMBEDDINGS_PROBE_K = 10;
+
+interface EntityEmbeddingsParams {
+ /** The requesting user, whose token authorizes the query. */
+ userId: string;
+ file_id: string;
+ /** The namespace to look in, such as an agent id. */
+ entity_id: string;
+ /** Any non-empty text; RAG needs one to rank chunks. */
+ query: string;
+ /** How long to wait for RAG before giving up; a stalled check rejects. */
+ timeoutMs?: number;
+}
+
+/**
+ * Whether RAG holds chunks of a file under an entity's namespace. RAG filters a query by
+ * file_id and also returns chunks owned by the requesting user, so a result counts only
+ * when its recorded owner is the entity itself.
+ * Rejects when RAG cannot be asked; the caller chooses the fallback.
+ */
+export async function hasEntityEmbeddings({
+ userId,
+ file_id,
+ entity_id,
+ query,
+ timeoutMs,
+}: EntityEmbeddingsParams): Promise<boolean> {
+ if (!process.env.RAG_API_URL) {
+ return false;
+ }
+ const { data } = await axios.post<Array<[{ metadata?: { user_id?: string } }, number]>>(
+ `${process.env.RAG_API_URL}/query`,
+ { file_id, query, k: ENTITY_EMBEDDINGS_PROBE_K, entity_id },
+ {
+ headers: {
+ Authorization: `Bearer ${generateShortLivedToken(userId)}`,
+ 'Content-Type': 'application/json',
+ },
+ ...(timeoutMs != null ? { timeout: timeoutMs } : {}),
+ },
+ );
+ return Array.isArray(data) && data.some(([chunk]) => chunk?.metadata?.user_id === entity_id);
+}
diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts
index 3ac2c521d3..837ff1f36f 100644
--- a/packages/data-provider/src/file-config.ts
+++ b/packages/data-provider/src/file-config.ts
@@ -667,6 +667,7 @@ export const fileConfigSchema = z.object({
fileContextSizeLimit: z.number().min(0).optional(),
fileContextCharLimit: z.number().min(0).optional(),
codeEnvLivenessSafeWindowMs: z.number().min(0).optional(),
+ legacySearchProbeTimeoutMs: z.number().min(1).optional(),
imageGeneration: z
.object({
percentage: z.number().min(0).max(100).optional(),
diff --git a/packages/data-schemas/src/methods/file.spec.ts b/packages/data-schemas/src/methods/file.spec.ts
index 4a867c5dcc..d504869c9e 100644
--- a/packages/data-schemas/src/methods/file.spec.ts
+++ b/packages/data-schemas/src/methods/file.spec.ts
@@ -47,6 +47,166 @@ describe('File Methods', () => {
await File.deleteMany({});
});
+ describe('adoptLegacySearchEmbeddings', () => {
+ const Agent = () => mongoose.models.Agent as mongoose.Model<unknown>;
+ const author = new mongoose.Types.ObjectId();
+
+ const createAgent = (id: string, searchFileIds: string[]) =>
+ Agent().create({
+ id,
+ name: id,
+ provider: 'openai',
+ model: 'gpt-4o',
+ author,
+ tool_resources: { [EToolResources.file_search]: { file_ids: searchFileIds } },
+ });
+
+ const createLegacyFile = (
+ fileId: string,
+ metadata?: Record<string, unknown>,
+ owner: mongoose.Types.ObjectId = author,
+ ) =>
+ File.create({
+ file_id: fileId,
+ user: owner,
+ filename: `${fileId}.pdf`,
+ filepath: `/uploads/${fileId}.pdf`,
+ type: 'application/pdf',
+ bytes: 1024,
+ embedded: true,
+ context: FileContext.agents,
+ ...(metadata ? { metadata } : {}),
+ });
+
+ afterEach(async () => {
+ await Agent().deleteMany({});
+ });
+
+ it('records the namespace of a verified legacy file', async () => {
+ const fileId = uuidv4();
+ await createAgent('agent_owner', [fileId]);
+ await createLegacyFile(fileId);
+
+ const adopted = await fileMethods.adoptLegacySearchEmbeddings({
+ files: [{ file_id: fileId, user: author.toString() }],
+ agentId: 'agent_owner',
+ });
+
+ expect(adopted).toEqual([fileId]);
+ const stored = (await File.findOne({ file_id: fileId }).lean()) as IMongoFile | null;
+ expect(stored?.metadata?.embeddedEntities).toEqual(['agent_owner']);
+ });
+
+ it('does not touch a tenant record that shares an untenanted file_id', async () => {
+ const fileId = uuidv4();
+ await File.collection.insertOne({
+ file_id: fileId,
+ user: author,
+ tenantId: 'tenant-b',
+ filename: `${fileId}.pdf`,
+ filepath: `/uploads/${fileId}.pdf`,
+ type: 'application/pdf',
+ bytes: 1024,
+ embedded: true,
+ context: FileContext.agents,
+ object: 'file',
+ usage: 0,
+ });
+ /* Inserted second, so an unscoped match would land on the tenant record first. */
+ await createLegacyFile(fileId);
+
+ const adopted = await fileMethods.adoptLegacySearchEmbeddings({
+ files: [{ file_id: fileId, user: author.toString() }],
+ agentId: 'agent_owner',
+ });
+
+ expect(adopted).toEqual([fileId]);
+ const tenantRecord = await File.collection.findOne({ file_id: fileId, tenantId: 'tenant-b' });
+ expect(tenantRecord?.metadata?.embeddedEntities).toBeUndefined();
+ const untenanted = (await File.findOne({
+ file_id: fileId,
+ tenantId: { $exists: false },
+ }).lean()) as IMongoFile | null;
+ expect(untenanted?.metadata?.embeddedEntities).toEqual(['agent_owner']);
+ });
+
+ it("does not touch another owner's record that shares the file_id", async () => {
+ const fileId = uuidv4();
+ const otherOwner = new mongoose.Types.ObjectId();
+ await createAgent('agent_owner', [fileId]);
+ await createLegacyFile(fileId);
+ await createLegacyFile(fileId, undefined, otherOwner);
+
+ const adopted = await fileMethods.adoptLegacySearchEmbeddings({
+ files: [{ file_id: fileId, user: author.toString() }],
+ agentId: 'agent_owner',
+ });
+
+ expect(adopted).toEqual([fileId]);
+ const foreign = (await File.findOne({
+ file_id: fileId,
+ user: otherOwner,
+ }).lean()) as IMongoFile | null;
+ expect(foreign?.metadata?.embeddedEntities).toBeUndefined();
+ });
+
+ it('reports a file a concurrent turn already adopted for the same agent', async () => {
+ const fileId = uuidv4();
+ await createAgent('agent_owner', [fileId]);
+ await createLegacyFile(fileId, { embeddedEntities: ['agent_owner'] });
+
+ const adopted = await fileMethods.adoptLegacySearchEmbeddings({
+ files: [{ file_id: fileId, user: author.toString() }],
+ agentId: 'agent_owner',
+ });
+
+ expect(adopted).toEqual([fileId]);
+ });
+
+ it('reports a committed adoption when a sibling write fails', async () => {
+ const adoptedId = uuidv4();
+ const failingId = uuidv4();
+ await createAgent('agent_owner', [adoptedId, failingId]);
+ await createLegacyFile(adoptedId);
+ await createLegacyFile(failingId);
+ const original = File.findOneAndUpdate.bind(File);
+ const spy = jest
+ .spyOn(File, 'findOneAndUpdate')
+ .mockImplementation(((filter: { file_id?: string }, ...rest: unknown[]) =>
+ filter.file_id === failingId
+ ? { lean: () => Promise.reject(new Error('write failed')) }
+ : (original as (...args: unknown[]) => unknown)(filter, ...rest)) as never);
+
+ try {
+ const adopted = await fileMethods.adoptLegacySearchEmbeddings({
+ files: [
+ { file_id: adoptedId, user: author.toString() },
+ { file_id: failingId, user: author.toString() },
+ ],
+ agentId: 'agent_owner',
+ });
+
+ expect(adopted).toEqual([adoptedId]);
+ } finally {
+ spy.mockRestore();
+ }
+ });
+
+ it('keeps the namespaces already recorded', async () => {
+ const fileId = uuidv4();
+ await createLegacyFile(fileId, { embeddedEntities: ['agent_source'] });
+
+ const adopted = await fileMethods.adoptLegacySearchEmbeddings({
+ files: [{ file_id: fileId, user: author.toString() }],
+ agentId: 'agent_owner',
+ });
+
+ expect(adopted).toEqual([fileId]);
+ const stored = (await File.findOne({ file_id: fileId }).lean()) as IMongoFile | null;
+ expect(stored?.metadata?.embeddedEntities).toEqual(['agent_source', 'agent_owner']);
+ });
+ });
+
describe('createFile', () => {
it('should create a new file with TTL', async () => {
const fileId = uuidv4();
diff --git a/packages/data-schemas/src/methods/file.ts b/packages/data-schemas/src/methods/file.ts
index 2c74b1c976..dc68912799 100644
--- a/packages/data-schemas/src/methods/file.ts
+++ b/packages/data-schemas/src/methods/file.ts
@@ -17,6 +17,13 @@ import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite';
import { escapeRegExp } from '~/utils/string';
import logger from '../config/winston';
+/** A loaded file record, named by the fields that identify it: `file_id` is not unique alone. */
+export type LegacySearchFileRef = {
+ file_id: string;
+ user: string;
+ tenantId?: string;
+};
+
export const DEFAULT_AVAILABLE_PROJECT_FILES_LIMIT = 20;
export const MAX_AVAILABLE_PROJECT_FILES_LIMIT = 50;
@@ -348,6 +355,10 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
file_id: string;
entityId: string;
}) => Promise<IMongoFile | null>;
+ adoptLegacySearchEmbeddings: (data: {
+ files: LegacySearchFileRef[];
+ agentId: string;
+ }) => Promise<string[]>;
updateFileUsage: (data: {
file_id: string;
inc?: number;
@@ -1222,6 +1233,54 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
).lean<IMongoFile>();
}
+ /**
+ * Records an agent's namespace on embedded files whose vectors the caller has confirmed are
+ * in it, keeping any namespaces already recorded. Each record is matched by its owner and
+ * tenant as well as its `file_id`, which is not unique on its own.
+ *
+ * @param data - Verified file records, and the agent whose namespace holds their vectors
+ * @returns The file ids that now record the agent's namespace
+ */
+ async function adoptLegacySearchEmbeddings(data: {
+ files: LegacySearchFileRef[];
+ agentId: string;
+ }): Promise<string[]> {
+ if (data.files.length === 0) {
+ return [];
+ }
+ const File = mongoose.models.File as Model<IMongoFile>;
+ /* An untenanted record is matched as untenanted: omitting the predicate would let a
+ * same-id record under a tenant match instead. Each write settles on its own, so a
+ * committed one is reported even when a sibling fails. */
+ const settled = await Promise.allSettled(
+ data.files.map((file) =>
+ File.findOneAndUpdate(
+ {
+ file_id: file.file_id,
+ user: file.user,
+ tenantId: file.tenantId ?? null,
+ context: FileContext.agents,
+ embedded: true,
+ },
+ { $addToSet: { 'metadata.embeddedEntities': data.agentId } },
+ { projection: { file_id: 1 } },
+ ).lean<Pick<IMongoFile, 'file_id'>>(),
+ ),
+ );
+ const adopted: string[] = [];
+ settled.forEach((result, index) => {
+ if (result.status === 'rejected') {
+ logger.error('[adoptLegacySearchEmbeddings] Failed to record a confirmed embedding', {
+ file_id: data.files[index].file_id,
+ error: (result.reason as Error)?.message,
+ });
+ } else if (result.value) {
+ adopted.push(result.value.file_id);
+ }
+ });
+ return adopted;
+ }
+
/**
* Increments the usage of a file identified by file_id.
* @param data - The data to update, must contain file_id and the increment value for usage
@@ -1569,6 +1628,7 @@ export function createFileMethods(mongoose: typeof import('mongoose')): {
updateFile,
updateFileCodeEnvRef,
addFileEmbeddedEntity,
+ adoptLegacySearchEmbeddings,
updateFileUsage,
deleteFile,
deleteFiles,