Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🔌 feat: Integrate Durable Skill Lifecycle Across Writers and Routes (#17043)

Dustin Healy73847f2db522
diff --git a/api/package.json b/api/package.json
index 27b076a7eb..b6e4ba7dce 100644
--- a/api/package.json
+++ b/api/package.json
@@ -7,7 +7,8 @@
"server-dev": "echo 'please run this from the root directory'",
"test": "cross-env NODE_ENV=test jest",
"b:test": "NODE_ENV=test bun jest",
- "test:ci": "jest --ci --logHeapUsage"
+ "test:ci": "jest --ci --logHeapUsage",
+ "recover-skill-upload": "node ./recover-skill-upload.js"
},
"repository": {
"type": "git",
diff --git a/api/recover-skill-upload.js b/api/recover-skill-upload.js
new file mode 100644
index 0000000000..0ecd0e114c
--- /dev/null
+++ b/api/recover-skill-upload.js
@@ -0,0 +1,26 @@
+require('module-alias/register');
+const mongoose = require('mongoose');
+const { createModels, createMethods, runAsSystem } = require('@librechat/data-schemas');
+const { runSkillUploadRecoveryCommand } = require('@librechat/api');
+
+createModels(mongoose);
+const methods = createMethods(mongoose);
+
+runSkillUploadRecoveryCommand(process.argv.slice(2), {
+ connect: () => require('./db/connect').connectDb(),
+ disconnect: () => mongoose.disconnect(),
+ runAsSystem,
+ getIntent: methods.getSkillUploadIntent,
+ listIntents: methods.listSkillUploadIntents,
+ recoverStoppedIntent: methods.recoverStoppedSkillUploadIntent,
+ recoverStoppedCleanupHold: methods.recoverStoppedSkillUploadCleanupHold,
+ write: console.log,
+ writeError: console.error,
+})
+ .then((code) => {
+ process.exitCode = code;
+ })
+ .catch(() => {
+ console.error('Skill upload recovery failed');
+ process.exitCode = 1;
+ });
diff --git a/api/server/controllers/SkillStatesController.js b/api/server/controllers/SkillStatesController.js
index 1afeb27141..febf8c6f1f 100644
--- a/api/server/controllers/SkillStatesController.js
+++ b/api/server/controllers/SkillStatesController.js
@@ -1,5 +1,5 @@
const mongoose = require('mongoose');
-const { logger } = require('@librechat/data-schemas');
+const { logger, filterExistingSkillIds } = require('@librechat/data-schemas');
const {
MAX_SKILL_STATES,
toSkillStatesRecord,
@@ -20,13 +20,8 @@ function buildPruneDeps(user) {
if (!Skill) {
return validIds;
}
- const existing = await Skill.find({ _id: { $in: validIds } })
- .select('_id')
- .lean();
- const deploymentIds = getDeploymentSkillIds()
- .map((id) => id.toString())
- .filter((id) => validIds.includes(id));
- return [...existing.map((doc) => doc._id.toString()), ...deploymentIds];
+ const deploymentIds = new Set(getDeploymentSkillIds().map((id) => id.toString()));
+ return filterExistingSkillIds(mongoose, validIds, (id) => deploymentIds.has(id));
},
findAccessibleSkillIds: async () =>
mergeDeploymentSkillIds(
diff --git a/api/server/experimental.js b/api/server/experimental.js
index ee6cf8c8b7..45029cc141 100644
--- a/api/server/experimental.js
+++ b/api/server/experimental.js
@@ -65,6 +65,7 @@ const { capabilityContextMiddleware } = require('./middleware/roles/capabilities
const createValidateImageRequest = require('./middleware/validateImageRequest');
const { startExpiredFileSweep } = require('./services/Files/process');
const { initializeGitHubSkillSync } = require('./services/Skills/sync');
+const { startSkillDeletionSweep } = require('./services/Skills/deletion');
const { initializeAgentTriggerService } = require('./services/Agents/triggers');
const { resumeAgentEventDetachedAction } = require('./services/Agents/detachedActionResume');
const {
@@ -499,6 +500,7 @@ if (cluster.isMaster) {
const appConfig = await getAppConfig();
initializeFileStorage(appConfig);
initializeGitHubSkillSync(appConfig);
+ startSkillDeletionSweep(appConfig.skillDeletion);
// Register configured tool-approval policy hooks (mirrors the standard startup path).
// Honors the `enabled` kill switch; hooks are base-config-only, registered process-wide.
// Read from the BASE config specifically — `appConfig` above (getAppConfig() with no
diff --git a/api/server/index.js b/api/server/index.js
index 535e7a46c5..4acad6ff88 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -78,6 +78,7 @@ const initializeOAuthReconnectManager = require('./services/initializeOAuthRecon
const { capabilityContextMiddleware } = require('./middleware/roles/capabilities');
const createValidateImageRequest = require('./middleware/validateImageRequest');
const { initializeGitHubSkillSync } = require('./services/Skills/sync');
+const { startSkillDeletionSweep } = require('./services/Skills/deletion');
const { initializeAgentTriggerService } = require('./services/Agents/triggers');
const { resumeAgentEventDetachedAction } = require('./services/Agents/detachedActionResume');
const { initializeScheduleEngine, recordExpiredScheduleApproval } = require('./services/Schedules');
@@ -262,6 +263,7 @@ const startServer = async () => {
additionalSkills: getDeploymentPluginSkills(),
});
initializeGitHubSkillSync(appConfig);
+ startSkillDeletionSweep(appConfig.skillDeletion);
startExpiredFileSweep({ appConfig, loadAppConfig: getAppConfig });
// Register any programmatic tool-approval policy hooks declared in
// `endpoints.agents.toolApproval.hooks`. Honor the `enabled` kill switch: when tool
diff --git a/api/server/routes/agents/__tests__/openapi.contract.spec.js b/api/server/routes/agents/__tests__/openapi.contract.spec.js
index 88e837b3bf..76176b51b2 100644
--- a/api/server/routes/agents/__tests__/openapi.contract.spec.js
+++ b/api/server/routes/agents/__tests__/openapi.contract.spec.js
@@ -32,6 +32,7 @@ const { createMulterInstance } = require('~/server/routes/files/multer');
jest.mock('~/server/services/Config', () => ({
checkCapability: jest.fn().mockResolvedValue(true),
+ getAppConfig: jest.fn(),
}));
const specPath = path.resolve(__dirname, '../../../../../packages/api/openapi/agents.openapi.json');
@@ -175,6 +176,7 @@ describe('Agents OpenAPI actual HTTP contract', () => {
const appConfig = {
fileStrategy: 'local',
+ skillDeletion: { finalizationEnabled: true },
paths: { uploads: uploadsPath },
fileConfig: {},
endpoints: {
@@ -195,6 +197,7 @@ describe('Agents OpenAPI actual HTTP contract', () => {
},
},
};
+ require('~/server/services/Config').getAppConfig.mockResolvedValue(appConfig);
const auth = createAgentManagementAuth({
findUser: (filter) => mongoose.models.User.findOne(filter),
getAppConfig: async () => appConfig,
diff --git a/api/server/routes/skills.js b/api/server/routes/skills.js
index 321d17d00c..0d8826d36b 100644
--- a/api/server/routes/skills.js
+++ b/api/server/routes/skills.js
@@ -4,9 +4,10 @@ const express = require('express');
const {
createImportHandler,
createSkillUploadHandler,
+ createSkillBlobReserver,
+ createSkillBufferWriter,
+ createSkillStorageResolver,
generateCheckAccess,
- getStorageMetadata,
- resolveRequestTenantId,
restoreTenantContextFromReq,
} = require('@librechat/api');
const { logger } = require('@librechat/data-schemas');
@@ -14,15 +15,15 @@ const {
PermissionBits,
PermissionTypes,
Permissions,
- FileContext,
mergeFileConfig,
} = require('librechat-data-provider');
const {
createSkill,
getSkillById,
- deleteSkill,
upsertSkillFile,
getSkillFileByPath,
+ reserveSkillUploadIntent,
+ transitionSkillUploadIntent,
getRoleByName,
} = require('~/models');
const { requireJwtAuth, canAccessSkillResource } = require('~/server/middleware');
@@ -30,10 +31,20 @@ const { grantPermission } = require('~/server/services/PermissionService');
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
const { createFileLimiters } = require('~/server/middleware/limiters/uploadLimiters');
const { maybeRunGitHubSkillSyncForRequest } = require('~/server/services/Skills/sync');
+const { deleteSkillForLegacy } = require('~/server/services/Skills/deletion');
const configMiddleware = require('~/server/middleware/config/app');
const { getFileStrategy } = require('~/server/utils/getFileStrategy');
const router = express.Router();
+const reserveSkillBlobForConfig = createSkillBlobReserver({
+ getFileStrategy,
+ getStrategyFunctions,
+});
+const resolveSkillStorageForConfig = createSkillStorageResolver({
+ getFileStrategy,
+ getStrategyFunctions,
+});
+const saveSkillBuffer = createSkillBufferWriter({ getStrategyFunctions });
// ---------------------------------------------------------------------------
// Multer: memory storage for skill imports (zip processed in-memory)
@@ -104,14 +115,7 @@ const handlers = getSkillsHandlers();
// ---------------------------------------------------------------------------
// File storage helper: resolve the active strategy's saveBuffer
// ---------------------------------------------------------------------------
-function resolveSkillStorage(req, { isImage = false } = {}) {
- const source = getFileStrategy(req.config, { context: FileContext.skill_file, isImage });
- const strategy = getStrategyFunctions(source);
- if (!strategy.saveBuffer) {
- throw new Error(`Storage backend "${source}" does not support file writes`);
- }
- return { saveBuffer: strategy.saveBuffer, source };
-}
+const resolveSkillStorage = (req, options) => resolveSkillStorageForConfig(req.config, options);
// ---------------------------------------------------------------------------
// Import handler (zip/md/skill → create skill + files)
@@ -122,25 +126,17 @@ const importHandler = createImportHandler({
}),
createSkill,
getSkillById,
- deleteSkill,
+ deleteSkill: deleteSkillForLegacy,
upsertSkillFile,
- saveBuffer: (req, { userId, buffer, fileName, basePath, isImage, tenantId }) => {
- const requestTenantId = tenantId ?? resolveRequestTenantId(req);
- const storage = resolveSkillStorage(req, { isImage });
- return storage
- .saveBuffer({ userId, buffer, fileName, basePath, tenantId: requestTenantId })
- .then((filepath) => ({
- filepath,
- source: storage.source,
- ...getStorageMetadata({ filepath, source: storage.source }),
- }));
- },
+ getSkillFileByPath,
+ reserveSkillUploadIntent,
+ transitionSkillUploadIntent,
+ reserveBlob: (req, options) => reserveSkillBlobForConfig(req.config, options),
+ saveBuffer: (_req, options) => saveSkillBuffer(options),
deleteFile: (req, file) => {
const { deleteFile } = getStrategyFunctions(file.source);
- if (deleteFile) {
- return deleteFile(req, file);
- }
- return Promise.resolve();
+ if (!deleteFile) throw new Error(`Skill storage backend ${file.source} cannot delete files`);
+ return deleteFile(req, file);
},
grantPermission,
});
@@ -154,6 +150,8 @@ const uploadFileHandler = createSkillUploadHandler({
upsertSkillFile,
resolveStorage: resolveSkillStorage,
getStrategyFunctions,
+ reserveSkillUploadIntent,
+ transitionSkillUploadIntent,
});
// ---------------------------------------------------------------------------
diff --git a/api/server/routes/skills.tenant.test.js b/api/server/routes/skills.tenant.test.js
index 4586e3cbd1..8c78a8897d 100644
--- a/api/server/routes/skills.tenant.test.js
+++ b/api/server/routes/skills.tenant.test.js
@@ -34,7 +34,8 @@ jest.mock('~/server/middleware/config/app', () => (req, _res, next) => {
jest.mock('~/server/services/Files/strategies', () => ({
getStrategyFunctions: jest.fn().mockReturnValue({
- saveBuffer: jest.fn().mockResolvedValue('/uploads/test/file.txt'),
+ saveBuffer: jest.fn(async ({ userId, fileName }) => `/uploads/${userId}/${fileName}`),
+ deleteFile: jest.fn().mockResolvedValue(undefined),
getDownloadStream: jest.fn().mockResolvedValue({
pipe: jest.fn(),
on: jest.fn(),
diff --git a/api/server/routes/skills.test.js b/api/server/routes/skills.test.js
index 545c98dafe..a91ca7f0bf 100644
--- a/api/server/routes/skills.test.js
+++ b/api/server/routes/skills.test.js
@@ -41,6 +41,7 @@ jest.mock('~/server/services/Config', () => ({
getCachedTools: jest.fn().mockResolvedValue({}),
getAppConfig: jest.fn().mockResolvedValue({
fileStrategy: 'local',
+ skillDeletion: { finalizationEnabled: true },
paths: { uploads: '/tmp/uploads', images: '/tmp/images' },
}),
}));
@@ -57,7 +58,7 @@ jest.mock('~/server/middleware/config/app', () => (req, _res, next) => {
jest.mock('~/server/services/Files/strategies', () => ({
getStrategyFunctions: jest.fn().mockReturnValue({
- saveBuffer: jest.fn().mockResolvedValue('/uploads/test/file.txt'),
+ saveBuffer: jest.fn(async ({ userId, fileName }) => `/uploads/${userId}/${fileName}`),
deleteFile: jest.fn().mockResolvedValue(undefined),
getDownloadStream: jest.fn().mockResolvedValue({
pipe: jest.fn(),
@@ -107,6 +108,7 @@ let mongoServer;
let skillRoutes;
let Skill;
let SkillFile;
+let SkillUploadIntent;
let AclEntry;
let AccessRole;
let User;
@@ -126,6 +128,7 @@ beforeAll(async () => {
const dbModels = require('~/db/models');
Skill = dbModels.Skill;
SkillFile = dbModels.SkillFile;
+ SkillUploadIntent = dbModels.SkillUploadIntent;
AclEntry = dbModels.AclEntry;
AccessRole = dbModels.AccessRole;
User = dbModels.User;
@@ -153,11 +156,14 @@ beforeAll(async () => {
currentTestUser = testUsers.owner;
skillRoutes = require('./skills');
app.use('/api/skills', skillRoutes);
+ const { getSkillStatesController } = require('~/server/controllers/SkillStatesController');
+ app.get('/api/skill-states-probe', getSkillStatesController);
});
afterEach(async () => {
await Skill.deleteMany({});
await SkillFile.deleteMany({});
+ await SkillUploadIntent.deleteMany({});
await AclEntry.deleteMany({});
currentTestUser = testUsers.owner;
mockFileConfig = undefined;
@@ -262,6 +268,22 @@ describe('Skill routes', () => {
});
afterEach(() => errSpy.mockRestore());
+ it('prunes a logically deleted Skill from user Skill states before ACL cleanup', async () => {
+ const created = await createSkillAsOwner({ name: 'state-fence-skill' });
+ expect(created.status).toBe(201);
+ const id = created.body._id;
+ await User.updateOne({ _id: testUsers.owner._id }, { $set: { skillStates: { [id]: false } } });
+
+ const before = await request(app).get('/api/skill-states-probe');
+ expect(before.status).toBe(200);
+ expect(before.body[id]).toBe(false);
+
+ await Skill.updateOne({ _id: id }, { $set: { deletingAt: new Date() } });
+ const after = await request(app).get('/api/skill-states-probe');
+ expect(after.status).toBe(200);
+ expect(after.body[id]).toBeUndefined();
+ });
+
describe('POST /api/skills', () => {
it('creates a skill and grants SKILL_OWNER ACL', async () => {
const res = await createSkillAsOwner();
@@ -506,13 +528,21 @@ describe('Skill routes', () => {
});
it('persists storage metadata for imported skill files', async () => {
- const savedFilepath =
- 'https://cdn.example.com/r/us-east-2/uploads/user123/imported-script.sh';
- const saveBuffer = jest.fn().mockResolvedValue(savedFilepath);
+ const getFileURL = jest.fn(
+ async ({ userId, fileName }) =>
+ `https://cdn.example.com/r/us-east-2/uploads/${userId}/${fileName}`,
+ );
+ const saveBuffer = jest.fn(async (params) => getFileURL(params));
const { getFileStrategy } = require('~/server/utils/getFileStrategy');
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
- getFileStrategy.mockReturnValueOnce('cloudfront');
- getStrategyFunctions.mockReturnValueOnce({ saveBuffer });
+ const defaultStrategy = getStrategyFunctions();
+ getFileStrategy.mockReturnValue('cloudfront');
+ const cloudfrontStrategy = {
+ saveBuffer,
+ deleteFile: jest.fn().mockResolvedValue(undefined),
+ getFileURL,
+ };
+ getStrategyFunctions.mockReturnValue(cloudfrontStrategy);
const zip = new JSZip();
zip.file(
@@ -532,6 +562,8 @@ describe('Skill routes', () => {
filename: 'imported-skill.skill',
contentType: 'application/zip',
});
+ getFileStrategy.mockReturnValue('local');
+ getStrategyFunctions.mockReturnValue(defaultStrategy);
expect(res.status).toBe(201);
expect(saveBuffer).toHaveBeenCalledWith(
@@ -540,6 +572,7 @@ describe('Skill routes', () => {
basePath: 'uploads',
}),
);
+ const savedFilepath = await saveBuffer.mock.results[0].value;
const savedFile = await SkillFile.findOne({
relativePath: 'scripts/imported-script.sh',
@@ -548,14 +581,14 @@ describe('Skill routes', () => {
expect.objectContaining({
filepath: savedFilepath,
source: 'cloudfront',
- storageKey: 'r/us-east-2/uploads/user123/imported-script.sh',
+ storageKey: new URL(savedFilepath).pathname.slice(1),
storageRegion: 'us-east-2',
}),
);
});
it('rolls back the skill, files, ACL, and stored blob when a bundled file fails', async () => {
- const saveBuffer = jest.fn().mockResolvedValue('/uploads/test/queries.sql');
+ const saveBuffer = jest.fn(async ({ userId, fileName }) => `/uploads/${userId}/${fileName}`);
const deleteFile = jest.fn().mockResolvedValue(undefined);
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
/** Both the write and the rollback delete resolve a strategy, so the
@@ -598,7 +631,7 @@ describe('Skill routes', () => {
expect(await AclEntry.countDocuments({ resourceType: ResourceType.SKILL })).toBe(0);
expect(deleteFile).toHaveBeenCalledWith(
expect.anything(),
- expect.objectContaining({ filepath: '/uploads/test/queries.sql' }),
+ expect.objectContaining({ filepath: await saveBuffer.mock.results[0].value }),
);
});
@@ -798,8 +831,8 @@ describe('Skill routes', () => {
const { updateSkillFileContent } = require('~/models');
const originalStrategy = getStrategyFunctions.getMockImplementation();
const stored = new Map();
- const saveBuffer = jest.fn(async ({ buffer, fileName }) => {
- const filepath = `/uploads/${fileName}`;
+ const saveBuffer = jest.fn(async ({ buffer, userId, fileName }) => {
+ const filepath = `/uploads/${userId}/${fileName}`;
stored.set(filepath, Buffer.from(buffer));
return filepath;
});
@@ -810,7 +843,11 @@ describe('Skill routes', () => {
}
return Readable.from([buffer]);
});
- getStrategyFunctions.mockReturnValue({ saveBuffer, getDownloadStream });
+ getStrategyFunctions.mockReturnValue({
+ saveBuffer,
+ getDownloadStream,
+ deleteFile: jest.fn().mockResolvedValue(undefined),
+ });
try {
const created = await createSkillAsOwner();
@@ -967,7 +1004,7 @@ describe('Skill routes', () => {
expect((await Skill.findById(skillId).lean()).version).toBe(capturedVersion + 1);
const winner = await SkillFile.findOne({ skillId, relativePath }).lean();
- const saveBuffer = jest.fn(async ({ fileName }) => `/uploads/${fileName}`);
+ const saveBuffer = jest.fn(async ({ userId, fileName }) => `/uploads/${userId}/${fileName}`);
const deleteFile = jest.fn(async () => undefined);
getStrategyFunctions.mockReturnValue({ saveBuffer, deleteFile });
try {
@@ -1019,7 +1056,7 @@ describe('Skill routes', () => {
};
let browserRevision;
const deleteFile = jest.fn(async () => undefined);
- const saveBuffer = jest.fn(async ({ fileName }) => {
+ const saveBuffer = jest.fn(async ({ userId, fileName }) => {
if (saveBuffer.mock.calls.length === 1) {
const browser = await request(app)
.post(`${url}/references/overlap.md`)
@@ -1032,7 +1069,7 @@ describe('Skill routes', () => {
expect(browser.status).toBe(200);
browserRevision = browser.body.file_id;
}
- return `/uploads/${fileName}`;
+ return `/uploads/${userId}/${fileName}`;
});
getStrategyFunctions.mockReturnValue({ saveBuffer, deleteFile });
try {
@@ -1048,8 +1085,8 @@ describe('Skill routes', () => {
}),
).rejects.toMatchObject({ code: 'SKILL_FILE_CONFLICT' });
expect(saveBuffer).toHaveBeenCalledTimes(2);
- const agentPath = `/uploads/${saveBuffer.mock.calls[0][0].fileName}`;
- const browserPath = `/uploads/${saveBuffer.mock.calls[1][0].fileName}`;
+ const agentPath = await saveBuffer.mock.results[0].value;
+ const browserPath = await saveBuffer.mock.results[1].value;
expect(await SkillFile.findOne({ skillId, relativePath }).lean()).toMatchObject({
file_id: browserRevision,
filepath: browserPath,
@@ -1086,10 +1123,10 @@ describe('Skill routes', () => {
};
await upsertSkillFile(input);
const deleteFile = jest.fn(async () => undefined);
- const saveBuffer = jest.fn(async ({ fileName }) => {
+ const saveBuffer = jest.fn(async ({ userId, fileName }) => {
// Another writer commits after the handler's preflight but before its atomic update.
await upsertSkillFile({ ...input, file_id: 'winner', filepath: '/uploads/winner' });
- return `/uploads/${fileName}`;
+ return `/uploads/${userId}/${fileName}`;
});
getStrategyFunctions.mockReturnValue({ saveBuffer, deleteFile });
try {
@@ -1107,21 +1144,19 @@ describe('Skill routes', () => {
filepath: '/uploads/winner',
});
expect(deleteFile).toHaveBeenCalledTimes(1);
- expect(deleteFile.mock.calls[0][1].filepath).toBe(
- `/uploads/${saveBuffer.mock.calls[0][0].fileName}`,
- );
+ expect(deleteFile.mock.calls[0][1].filepath).toBe(await saveBuffer.mock.results[0].value);
} finally {
getStrategyFunctions.mockImplementation(originalStrategy);
}
});
- it('does not delete committed file bytes if the parent version bump fails', async () => {
+ it('acknowledges committed file bytes if the parent version bump fails', async () => {
const created = await createSkillAsOwner();
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
const originalStrategy = getStrategyFunctions.getMockImplementation();
const deleteFile = jest.fn(async () => undefined);
getStrategyFunctions.mockReturnValue({
- saveBuffer: jest.fn(async () => '/uploads/committed'),
+ saveBuffer: jest.fn(async ({ userId, fileName }) => `/uploads/${userId}/${fileName}`),
deleteFile,
});
const bump = jest
@@ -1135,9 +1170,10 @@ describe('Skill routes', () => {
filename: 'committed.md',
contentType: 'text/markdown',
});
- expect(response.status).toBe(500);
+ expect(response.status).toBe(200);
+ const committedPath = await getStrategyFunctions().saveBuffer.mock.results[0].value;
expect(await SkillFile.findOne({ skillId: created.body._id }).lean()).toMatchObject({
- filepath: '/uploads/committed',
+ filepath: committedPath,
});
expect(deleteFile).not.toHaveBeenCalled();
} finally {
@@ -1146,14 +1182,14 @@ describe('Skill routes', () => {
}
});
- it('cleans up the superseded blob when a replacement commits but the parent bump fails', async () => {
+ it('acknowledges a replacement and queues the superseded blob if the parent bump fails', async () => {
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
const { upsertSkillFile } = require('~/models');
const originalStrategy = getStrategyFunctions.getMockImplementation();
const created = await createSkillAsOwner();
const skillId = created.body._id;
const oldPath = '/uploads/previous';
- const newPath = '/uploads/replacement';
+ let newPath;
const stored = new Set([oldPath]);
await upsertSkillFile({
skillId,
@@ -1169,7 +1205,8 @@ describe('Skill routes', () => {
const deleteFile = jest.fn(async (_req, { filepath }) => {
stored.delete(filepath);
});
- const saveBuffer = jest.fn(async () => {
+ const saveBuffer = jest.fn(async ({ userId, fileName }) => {
+ newPath = `/uploads/${userId}/${fileName}`;
stored.add(newPath);
return newPath;
});
@@ -1186,18 +1223,19 @@ describe('Skill routes', () => {
filename: 'committed.md',
contentType: 'text/markdown',
});
- expect(response.status).toBe(500);
+ expect(response.status).toBe(200);
expect(await SkillFile.findOne({ skillId }).lean()).toMatchObject({
filepath: newPath,
file_id: expect.not.stringMatching('original'),
});
expect(saveBuffer).toHaveBeenCalledTimes(1);
- expect(deleteFile).toHaveBeenCalledTimes(1);
- expect(deleteFile.mock.calls[0][1]).toMatchObject({
- filepath: oldPath,
- user: testUsers.editor._id.toString(),
+ expect(deleteFile).not.toHaveBeenCalled();
+ expect(await SkillUploadIntent.findOne({ fileId: 'original' }).lean()).toMatchObject({
+ state: 'committed',
+ storageKey: oldPath,
+ cleanupRequestedAt: expect.any(Date),
});
- expect(stored.has(oldPath)).toBe(false);
+ expect(stored.has(oldPath)).toBe(true);
expect(stored.has(newPath)).toBe(true);
} finally {
bump.mockRestore();
diff --git a/api/server/services/Endpoints/agents/skillDeps.js b/api/server/services/Endpoints/agents/skillDeps.js
index f4155b98b5..6d1034e755 100644
--- a/api/server/services/Endpoints/agents/skillDeps.js
+++ b/api/server/services/Endpoints/agents/skillDeps.js
@@ -22,12 +22,12 @@ const {
createDeploymentSkillMethods,
createSkillFileSaver,
createSkillManagementFileSaver,
+ createSkillStorageResolver,
isDeploymentSkillFileSource,
getDeploymentSkillDownloadStream,
} = require('@librechat/api');
const {
Permissions,
- FileContext,
ResourceType,
PermissionBits,
AccessRoleIds,
@@ -39,6 +39,7 @@ const {
const { checkPermission, grantPermission } = require('~/server/services/PermissionService');
const { getFileStrategy } = require('~/server/utils/getFileStrategy');
const db = require('~/models');
+const { deleteSkillForLegacy } = require('~/server/services/Skills/deletion');
const deploymentSkillMethods = createDeploymentSkillMethods({
getSkillById: db.getSkillById,
@@ -68,20 +69,19 @@ function getSkillStrategyFunctions(source) {
return getStrategyFunctions(source);
}
-function resolveSkillStorage(req, { isImage = false } = {}) {
- const source = getFileStrategy(req.config, { context: FileContext.skill_file, isImage });
- const strategy = getStrategyFunctions(source);
- if (!strategy.saveBuffer) {
- throw new Error(`Storage backend "${source}" does not support file writes`);
- }
- return { saveBuffer: strategy.saveBuffer, source };
-}
+const resolveSkillStorageForConfig = createSkillStorageResolver({
+ getFileStrategy,
+ getStrategyFunctions,
+});
+const resolveSkillStorage = (req, options) => resolveSkillStorageForConfig(req.config, options);
const skillFileSaveDeps = {
getSkillFileByPath: db.getSkillFileByPath,
upsertSkillFile: db.upsertSkillFile,
resolveStorage: resolveSkillStorage,
getStrategyFunctions,
+ reserveSkillUploadIntent: db.reserveSkillUploadIntent,
+ transitionSkillUploadIntent: db.transitionSkillUploadIntent,
};
const saveSkillFileContent = createSkillFileSaver(skillFileSaveDeps);
const saveSkillManagementFileContent = createSkillManagementFileSaver(skillFileSaveDeps);
@@ -328,7 +328,7 @@ const skillToolDeps = {
getAuthorSkillByName,
createSkill: db.createSkill,
updateSkill: db.updateSkill,
- deleteSkill: db.deleteSkill,
+ deleteSkill: deleteSkillForLegacy,
canCreateSkill,
canEditSkill,
grantSkillOwner,
diff --git a/api/server/services/Endpoints/agents/skillDeps.spec.js b/api/server/services/Endpoints/agents/skillDeps.spec.js
index c107a52d75..7c0464bc45 100644
--- a/api/server/services/Endpoints/agents/skillDeps.spec.js
+++ b/api/server/services/Endpoints/agents/skillDeps.spec.js
@@ -50,6 +50,12 @@ jest.mock('@librechat/api', () => ({
createDeploymentSkillMethods: (...args) => mockCreateDeploymentSkillMethods(...args),
createSkillFileSaver: (...args) => mockCreateSkillFileSaver(...args),
createSkillManagementFileSaver: (...args) => mockCreateSkillManagementFileSaver(...args),
+ createSkillStorageResolver:
+ ({ getFileStrategy, getStrategyFunctions }) =>
+ (config, { isImage }) => {
+ const source = getFileStrategy(config, { context: 'skill_file', isImage });
+ return { source, saveBuffer: getStrategyFunctions(source).saveBuffer };
+ },
enrichWithSkillConfigurable: jest.fn(),
getDeploymentSkillDownloadStream: jest.fn(),
getStorageMetadata: (...args) => mockGetStorageMetadata(...args),
@@ -74,6 +80,10 @@ jest.mock('~/server/services/PermissionService', () => ({
grantPermission: jest.fn(),
}));
+jest.mock('~/server/services/Skills/deletion', () => ({
+ deleteSkillForLegacy: jest.fn(),
+}));
+
jest.mock('~/server/utils/getFileStrategy', () => ({
getFileStrategy: (...args) => mockGetFileStrategy(...args),
}));
diff --git a/api/server/services/Skills/deletion.js b/api/server/services/Skills/deletion.js
new file mode 100644
index 0000000000..209ef0c5a8
--- /dev/null
+++ b/api/server/services/Skills/deletion.js
@@ -0,0 +1,19 @@
+const {
+ logger,
+ runAsSystem,
+ createSkillDeletionPersistenceAdapter,
+} = require('@librechat/data-schemas');
+const { createSkillDeletionService } = require('@librechat/api');
+const db = require('~/models');
+const { getAppConfig } = require('~/server/services/Config');
+const { getStrategyFunctions } = require('~/server/services/Files/strategies');
+
+const service = createSkillDeletionService({
+ persistence: { ...db, ...createSkillDeletionPersistenceAdapter(db) },
+ loadConfig: () => getAppConfig({ baseOnly: true }),
+ getStrategyFunctions,
+ runAsSystem,
+ logger,
+});
+
+module.exports = service;
diff --git a/api/server/services/Skills/handlers.js b/api/server/services/Skills/handlers.js
index f224cac34e..3699546693 100644
--- a/api/server/services/Skills/handlers.js
+++ b/api/server/services/Skills/handlers.js
@@ -1,7 +1,7 @@
const { createSkillsHandlers } = require('@librechat/api');
const { isValidObjectIdString } = require('@librechat/data-schemas');
const { PermissionBits } = require('librechat-data-provider');
-const { createSkill, updateSkill, deleteSkill, deleteSkillFile } = require('~/models');
+const { createSkill, updateSkill, deleteSkillFile } = require('~/models');
const {
findAccessibleResources,
findPubliclyAccessibleResources,
@@ -13,6 +13,7 @@ const {
withDeploymentSkillIds,
getSkillStrategyFunctions,
} = require('~/server/services/Endpoints/agents/skillDeps');
+const { deleteSkillById, deleteSkillForLegacy } = require('./deletion');
function getSkillsHandlers() {
const skillDbMethods = getSkillDbMethods();
@@ -21,7 +22,8 @@ function getSkillsHandlers() {
getSkillById: skillDbMethods.getSkillById,
listSkillsByAccess: skillDbMethods.listSkillsByAccess,
updateSkill,
- deleteSkill,
+ deleteSkill: deleteSkillForLegacy,
+ deleteSkillById,
listSkillFiles: skillDbMethods.listSkillFiles,
deleteSkillFile,
getSkillFileByPath: skillDbMethods.getSkillFileByPath,
diff --git a/api/server/services/Skills/sync.js b/api/server/services/Skills/sync.js
index f8d45fa130..c638030f33 100644
--- a/api/server/services/Skills/sync.js
+++ b/api/server/services/Skills/sync.js
@@ -1,8 +1,8 @@
-const { FileContext } = require('librechat-data-provider');
const {
- getStorageMetadata,
createTermsBlockCheck,
skipUntilTermsAccepted,
+ createSkillBlobReserver,
+ createSkillBufferWriter,
createGitHubSkillSyncRunner,
createSkillSyncTriggerOrchestrator,
startGitHubSkillSyncScheduler,
@@ -12,10 +12,16 @@ const db = require('~/models');
const { getAppConfig } = require('~/server/services/Config');
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
const { getFileStrategy } = require('~/server/utils/getFileStrategy');
+const { deleteSkillForLegacy } = require('./deletion');
const isTermsAcceptanceRequired = createTermsBlockCheck({ getAppConfig });
const SYSTEM_USER_ID = '000000000000000000000000';
+const reserveSkillBlobForConfig = createSkillBlobReserver({
+ getFileStrategy,
+ getStrategyFunctions,
+});
+const saveSkillBuffer = createSkillBufferWriter({ getStrategyFunctions });
let appConfigRef;
let runner;
@@ -39,16 +45,6 @@ async function getSyncConfig(loadAppConfig = loadCurrentAppConfig) {
return appConfig?.skillSync;
}
-async function resolveSkillStorage({ isImage = false, loadAppConfig = loadCurrentAppConfig } = {}) {
- const appConfig = await loadAppConfig();
- const source = getFileStrategy(appConfig, { context: FileContext.skill_file, isImage });
- const strategy = getStrategyFunctions(source);
- if (!strategy.saveBuffer) {
- throw new Error(`Storage backend "${source}" does not support file writes`);
- }
- return { source, saveBuffer: strategy.saveBuffer };
-}
-
async function getSyntheticReq({ userId = SYSTEM_USER_ID, tenantId, loadAppConfig } = {}) {
const appConfig = await (loadAppConfig ?? loadCurrentAppConfig)();
return {
@@ -98,8 +94,14 @@ function createRunner({ getConfig, loadAppConfig, allowServerCredentials = true
listSkillFiles: db.listSkillFiles,
getSkillFileByPath: db.getSkillFileByPath,
upsertSkillFile: db.upsertSkillFile,
+ reserveSkillUploadIntent: db.reserveSkillUploadIntent,
+ transitionSkillUploadIntent: db.transitionSkillUploadIntent,
+ queueCommittedSkillUploadCleanup: db.queueCommittedSkillUploadCleanup,
+ markSkillUploadIntentCleaned: db.markSkillUploadIntentCleaned,
+ finishSkillUploadCleanupHold: db.finishSkillUploadCleanupHold,
deleteSkillFile: db.deleteSkillFile,
- deleteSkill: db.deleteSkill,
+ deleteSkill: deleteSkillForLegacy,
+ reserveBlob: (options) => reserveSkillBlobForConfig(resolveAppConfig(), options),
grantPermission: async ({
principalType,
principalId,
@@ -132,25 +134,11 @@ function createRunner({ getConfig, loadAppConfig, allowServerCredentials = true
role._id,
);
},
- saveBuffer: async ({ userId, buffer, fileName, basePath, isImage, tenantId }) => {
- const storage = await resolveSkillStorage({ isImage, loadAppConfig: resolveAppConfig });
- const filepath = await storage.saveBuffer({
- userId: userId ?? SYSTEM_USER_ID,
- buffer,
- fileName,
- basePath,
- tenantId,
- });
- return {
- filepath,
- source: storage.source,
- ...getStorageMetadata({ filepath, source: storage.source }),
- };
- },
+ saveBuffer: saveSkillBuffer,
deleteFile: async (file) => {
const strategy = getStrategyFunctions(file.source);
if (!strategy.deleteFile) {
- return;
+ throw new Error(`Skill storage backend ${file.source} cannot delete files`);
}
await strategy.deleteFile(
await getSyntheticReq({
diff --git a/api/server/services/Skills/sync.test.js b/api/server/services/Skills/sync.test.js
index 04728b1dba..a9403b89e9 100644
--- a/api/server/services/Skills/sync.test.js
+++ b/api/server/services/Skills/sync.test.js
@@ -16,6 +16,9 @@ jest.mock('@librechat/api', () => {
return {
createTermsBlockCheck: actualApi.createTermsBlockCheck,
skipUntilTermsAccepted: actualApi.skipUntilTermsAccepted,
+ createSkillDeletionService: actualApi.createSkillDeletionService,
+ createSkillBlobReserver: actualApi.createSkillBlobReserver,
+ createSkillBufferWriter: actualApi.createSkillBufferWriter,
createSkillSyncTriggerOrchestrator: actualApi.createSkillSyncTriggerOrchestrator,
createGitHubSkillSyncRunner: jest.fn((deps) => {
mockRunnerDeps = deps;
@@ -60,6 +63,7 @@ jest.mock('@librechat/api', () => {
});
jest.mock('@librechat/data-schemas', () => ({
+ createSkillDeletionPersistenceAdapter: jest.fn(() => ({})),
logger: {
error: jest.fn(),
warn: jest.fn(),
diff --git a/docs/skills-management-api.md b/docs/skills-management-api.md
index 170fb15350..5cf35f6abd 100644
--- a/docs/skills-management-api.md
+++ b/docs/skills-management-api.md
@@ -52,8 +52,13 @@ permissions. Inaccessible and cross-tenant IDs return the same 404 envelope.
Whole-Skill deletion requires DELETE access and the same Skills USE/CREATE role permissions as
other mutations. It returns `{ "id": "...", "deleted": true, "cleanupComplete": true }` on
-complete cleanup. If the existing deletion handler removes the Skill but cannot finish dependent
-cleanup, it returns `cleanupComplete: false`; that result does not promise an automatic retry.
+complete cleanup. If cleanup is still pending, it returns `cleanupComplete: false`. The Skill is
+hidden immediately, and the background sweeper retries durable cleanup after transient failures.
+Physical finalization requires `skillDeletion.finalizationEnabled: true` after older Skill writers
+have drained; without it, deletion stays pending.
+Cleanup reads file metadata and upload intents in cursor batches bounded by
+`skillDeletion.batchSize` (default 100), settling each batch before fetching the next.
+File content is not loaded, and failed provider deletions retain their persisted references for retry.
Text-file writes accept `{ "content": "replacement text" }`, capped at 1 MiB of UTF-8 data.
Use a relative path such as `references/guide.md`. Absolute paths, traversal, and NUL bytes are
@@ -63,6 +68,8 @@ a version precondition; serialize writers to the same path. After a file write,
before a versioned metadata/body update: file writes also increment the parent version. Storage selection stays server-controlled. File JSON reads omit content for
binary or oversized files; raw download mode is not supported on this surface. File deletion completes
storage cleanup before returning success and reports a conflict if another writer replaced the path.
+Replacing a file queues the previous blob for durable background cleanup. If the replacement fails,
+the sweeper retains any blob still referenced by the current file row.
Git-synced and deployment-provided Skills are readable but cannot be changed through this API.
Change their upstream source instead. Browser routes keep their existing behavior.
diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml
index e31c765ead..95ee36ebb5 100644
--- a/e2e/config/librechat.e2e.yaml
+++ b/e2e/config/librechat.e2e.yaml
@@ -4,6 +4,12 @@
version: 1.3.11
cache: true
+# The isolated E2E fleet has only upload-intent-aware writers, so it can
+# exercise whole-Skill finalization instead of the mixed-version rollout gate.
+skillDeletion:
+ finalizationEnabled: true
+ sweepIntervalMs: 1000
+
interface:
# Isolated test fleet contains only ownership-aware replicas.
runningChatRename: true
diff --git a/e2e/specs/mock/skill-file-authoring.spec.ts b/e2e/specs/mock/skill-file-authoring.spec.ts
index 529a2991bf..7614bf460f 100644
--- a/e2e/specs/mock/skill-file-authoring.spec.ts
+++ b/e2e/specs/mock/skill-file-authoring.spec.ts
@@ -124,5 +124,11 @@ test.describe('skill file authoring tools', () => {
await page.goto(`/skills/${editedSkill._id}`, { timeout: 10000 });
await expect(page.getByRole('heading', { level: 1, name: skillName })).toBeVisible();
await expect(page.getByText(EDITED_DESCRIPTION)).toBeVisible();
+
+ await page.getByRole('button', { name: 'Delete' }).click();
+ await page.getByRole('dialog').getByRole('button', { name: 'Delete' }).click();
+ await expect(page).toHaveURL((url) => url.pathname === '/skills');
+ const token = await getAccessToken(page);
+ await expect.poll(async () => findSkill(page, skillName, token)).toBeNull();
});
});
diff --git a/e2e/specs/mock/skill-lifecycle-live.spec.ts b/e2e/specs/mock/skill-lifecycle-live.spec.ts
new file mode 100644
index 0000000000..ce68a1791b
--- /dev/null
+++ b/e2e/specs/mock/skill-lifecycle-live.spec.ts
@@ -0,0 +1,399 @@
+import fs from 'fs';
+import path from 'path';
+import mongoose from 'mongoose';
+import JSZip from 'jszip';
+import { expect, test } from '@playwright/test';
+import { createMethods, createModels, tenantStorage } from '@librechat/data-schemas';
+import {
+ managementTenantId,
+ managementUserId,
+ startManagementOidc,
+} from '../../setup/agent-management';
+import { applyRuntimeEnv } from '../../setup/runtimeEnv';
+import {
+ getAccessToken,
+ NEW_CHAT_PATH,
+ MOCK_ENDPOINTS,
+ selectMockEndpoint,
+ uploadViaUnifiedButton,
+ isAgentsStream,
+} from './helpers';
+
+const api = '/api/agents/v1/skills';
+const diskPath = (filepath: string) => {
+ expect(filepath).toMatch(/^\/uploads\/[^/]+\/[^/]+$/);
+ return path.resolve('uploads', filepath.slice('/uploads/'.length));
+};
+
+test.describe('full-stack Skill lifecycle confirmation', () => {
+ let oidc: Awaited<ReturnType<typeof startManagementOidc>>;
+ let headers: { Authorization: string };
+ let methods: ReturnType<typeof createMethods>;
+
+ test.beforeAll(async () => {
+ oidc = await startManagementOidc();
+ headers = { Authorization: `Bearer ${oidc.token}` };
+ applyRuntimeEnv();
+ await mongoose.connect(process.env.MONGO_URI!);
+ createModels(mongoose);
+ methods = createMethods(mongoose);
+ await tenantStorage.run({ tenantId: managementTenantId }, async () => {
+ await methods.initializeRoles();
+ await methods.seedDefaultRoles();
+ await methods.seedSystemGrants();
+ await mongoose.models.User.updateOne(
+ { _id: managementUserId },
+ {
+ $set: {
+ name: 'Lifecycle E2E',
+ email: '[email protected]',
+ emailVerified: true,
+ provider: 'local',
+ role: 'ADMIN',
+ tenantId: managementTenantId,
+ },
+ },
+ { upsert: true },
+ );
+ });
+ });
+
+ test.afterAll(async () => {
+ await mongoose.disconnect();
+ await oidc?.close();
+ });
+
+ test('machine API creates, lists, reads, updates, rejects stale writes and deletes individual files', async ({
+ request,
+ }) => {
+ const name = `full-crud-${Date.now()}`;
+ const created = await request.post(api, {
+ headers,
+ data: { name, description: 'Original description.', body: 'Original body.' },
+ });
+ expect(created.status(), await created.text()).toBe(201);
+ const initial = await created.json();
+ const url = `${api}/${initial.id}`;
+ const listed = await request.get(`${api}?limit=100`, { headers });
+ expect(listed.status()).toBe(200);
+ expect((await listed.json()).data).toContainEqual(expect.objectContaining({ id: initial.id }));
+ const get = await request.get(url, { headers });
+ expect(get.status()).toBe(200);
+ expect(await get.json()).toMatchObject({
+ name,
+ body: 'Original body.',
+ version: initial.version,
+ });
+
+ const updated = await request.patch(url, {
+ headers,
+ data: {
+ expectedVersion: initial.version,
+ description: 'Updated description.',
+ body: 'Updated body.',
+ },
+ });
+ expect(updated.status(), await updated.text()).toBe(200);
+ expect(await updated.json()).toMatchObject({
+ description: 'Updated description.',
+ body: 'Updated body.',
+ version: initial.version + 1,
+ });
+ const stale = await request.patch(url, {
+ headers,
+ data: { expectedVersion: initial.version, body: 'Must not clobber.' },
+ });
+ expect(stale.status(), await stale.text()).toBe(409);
+ expect((await (await request.get(url, { headers })).json()).body).toBe('Updated body.');
+
+ const fileUrl = `${url}/files/references/live.txt`;
+ expect(
+ (await request.put(fileUrl, { headers, data: { content: 'First bytes.' } })).status(),
+ ).toBe(200);
+ const first = await tenantStorage.run({ tenantId: managementTenantId }, () =>
+ methods.getSkillFileByPath(initial.id, 'references/live.txt'),
+ );
+ expect(first).not.toBeNull();
+ expect(fs.readFileSync(diskPath(first!.filepath), 'utf8')).toBe('First bytes.');
+ expect(
+ (await request.put(fileUrl, { headers, data: { content: 'Second bytes.' } })).status(),
+ ).toBe(200);
+ const second = await tenantStorage.run({ tenantId: managementTenantId }, () =>
+ methods.getSkillFileByPath(initial.id, 'references/live.txt'),
+ );
+ expect(second!.file_id).not.toBe(first!.file_id);
+ expect(fs.readFileSync(diskPath(second!.filepath), 'utf8')).toBe('Second bytes.');
+ await expect.poll(() => fs.existsSync(diskPath(first!.filepath))).toBe(false);
+ await expect
+ .poll(async () => {
+ const intent = await tenantStorage.run({ tenantId: managementTenantId }, () =>
+ methods.getSkillUploadIntent(first!.file_id),
+ );
+ return { cleaned: intent?.cleanedAt != null, released: intent?.claimOwner == null };
+ })
+ .toEqual({ cleaned: true, released: true });
+ expect((await (await request.get(fileUrl, { headers })).json()).content).toBe('Second bytes.');
+ expect((await (await request.get(`${url}/files`, { headers })).json()).data).toContainEqual(
+ expect.objectContaining({ relativePath: 'references/live.txt' }),
+ );
+
+ const removed = await request.delete(fileUrl, { headers });
+ expect(removed.status(), await removed.text()).toBe(200);
+ expect(await removed.json()).toEqual({
+ skillId: initial.id,
+ relativePath: 'references/live.txt',
+ deleted: true,
+ });
+ expect(fs.existsSync(diskPath(second!.filepath))).toBe(false);
+ expect((await request.get(fileUrl, { headers })).status()).toBe(404);
+ expect((await (await request.get(`${url}/files`, { headers })).json()).data).toEqual([]);
+ expect((await request.delete(fileUrl, { headers })).status()).toBe(404);
+ expect((await request.delete(url, { headers })).status()).toBe(200);
+ expect((await request.get(url, { headers })).status()).toBe(404);
+ });
+
+ test('authorized management deletion cleans a seeded other-author file through the real Local adapter', async ({
+ request,
+ }) => {
+ const created = await request.post(api, {
+ headers,
+ data: {
+ name: `cross-author-${Date.now()}`,
+ description: 'Cross-author cleanup.',
+ body: 'Instructions.',
+ },
+ });
+ expect(created.status()).toBe(201);
+ const { id } = await created.json();
+ const url = `${api}/${id}/files/references/owned.txt`;
+ expect(
+ (await request.put(url, { headers, data: { content: 'Recorded owner bytes.' } })).status(),
+ ).toBe(200);
+ const owner = new mongoose.Types.ObjectId();
+ let movedPath = '';
+ await tenantStorage.run({ tenantId: managementTenantId }, async () => {
+ const file = await methods.getSkillFileByPath(id, 'references/owned.txt');
+ expect(file).not.toBeNull();
+ const persistedPath = file!.filepath.replace(
+ `/uploads/${managementUserId}/`,
+ `/uploads/${owner}/`,
+ );
+ movedPath = diskPath(persistedPath);
+ await fs.promises.mkdir(path.dirname(movedPath), { recursive: true });
+ await fs.promises.rename(diskPath(file!.filepath), movedPath);
+ // Fixture represents a pre-existing file authored by another user in the same organization.
+ await mongoose.models.SkillFile.updateOne(
+ { file_id: file!.file_id },
+ {
+ $set: { author: owner, filepath: persistedPath },
+ },
+ );
+ });
+ const removed = await request.delete(url, { headers });
+ expect(removed.status(), await removed.text()).toBe(200);
+ expect(fs.existsSync(movedPath)).toBe(false);
+ expect((await request.get(url, { headers })).status()).toBe(404);
+ // A subsequent request still uses the authenticated manager, not the cloned storage owner.
+ const next = await request.put(`${api}/${id}/files/references/next.txt`, {
+ headers,
+ data: { content: 'Manager bytes.' },
+ });
+ expect(next.status(), await next.text()).toBe(200);
+ const nextFile = await tenantStorage.run({ tenantId: managementTenantId }, () =>
+ methods.getSkillFileByPath(id, 'references/next.txt'),
+ );
+ expect(nextFile!.author.toString()).toBe(managementUserId);
+ expect((await request.delete(`${api}/${id}`, { headers })).status()).toBe(200);
+ expect(fs.existsSync(diskPath(nextFile!.filepath))).toBe(false);
+ });
+
+ test('browser-authenticated multipart binary upload reads exact bytes, replaces by revision and deletes', async ({
+ page,
+ request,
+ }) => {
+ await page.goto(NEW_CHAT_PATH);
+ const token = await getAccessToken(page);
+ const browserHeaders = { Authorization: `Bearer ${token}` };
+ const created = await request.post('/api/skills', {
+ headers: browserHeaders,
+ data: {
+ name: `binary-lifecycle-${Date.now()}`,
+ description: 'Binary file lifecycle.',
+ body: 'Binary attachment test.',
+ },
+ });
+ expect(created.status(), await created.text()).toBe(201);
+ const { _id: id } = await created.json();
+ const url = `/api/skills/${id}/files`;
+ const firstBytes = Buffer.from([0, 1, 2, 3, 255, 128, 0, 42]);
+ const uploaded = await request.post(url, {
+ headers: browserHeaders,
+ multipart: {
+ relativePath: 'assets/payload.bin',
+ file: { name: 'payload.bin', mimeType: 'application/octet-stream', buffer: firstBytes },
+ },
+ });
+ expect(uploaded.status(), await uploaded.text()).toBe(200);
+ const first = await uploaded.json();
+ expect(fs.readFileSync(diskPath(first.filepath))).toEqual(firstBytes);
+ const binary = await request.get(`${url}/assets/payload.bin`, { headers: browserHeaders });
+ expect(binary.status()).toBe(200);
+ expect(await binary.json()).toMatchObject({ isBinary: true, bytes: firstBytes.length });
+ const raw = await request.get(`${url}/assets/payload.bin?raw=true`, {
+ headers: browserHeaders,
+ });
+ expect(raw.status()).toBe(200);
+ expect(await raw.body()).toEqual(firstBytes);
+
+ const nextBytes = Buffer.from([0, 42, 255, 7]);
+ const replaced = await request.post(`${url}/assets/payload.bin`, {
+ headers: browserHeaders,
+ multipart: {
+ relativePath: 'assets/payload.bin',
+ expectedFileId: first.file_id,
+ file: { name: 'payload.bin', mimeType: 'application/octet-stream', buffer: nextBytes },
+ },
+ });
+ expect(replaced.status(), await replaced.text()).toBe(200);
+ const second = await replaced.json();
+ expect(second.file_id).not.toBe(first.file_id);
+ expect(fs.readFileSync(diskPath(second.filepath))).toEqual(nextBytes);
+ await expect.poll(() => fs.existsSync(diskPath(first.filepath))).toBe(false);
+ const stale = await request.post(`${url}/assets/payload.bin`, {
+ headers: browserHeaders,
+ multipart: {
+ relativePath: 'assets/payload.bin',
+ expectedFileId: first.file_id,
+ file: { name: 'payload.bin', mimeType: 'application/octet-stream', buffer: firstBytes },
+ },
+ });
+ expect(stale.status(), await stale.text()).toBe(409);
+ expect(fs.readFileSync(diskPath(second.filepath))).toEqual(nextBytes);
+ expect(
+ (await request.delete(`${url}/assets/payload.bin`, { headers: browserHeaders })).status(),
+ ).toBe(200);
+ expect(fs.existsSync(diskPath(second.filepath))).toBe(false);
+ expect(
+ (await request.get(`${url}/assets/payload.bin`, { headers: browserHeaders })).status(),
+ ).toBe(404);
+ expect((await request.delete(`/api/skills/${id}`, { headers: browserHeaders })).status()).toBe(
+ 200,
+ );
+ });
+
+ test('imports an archive with text and binary attachments and physically cleans all files on deletion', async ({
+ page,
+ request,
+ }) => {
+ await page.goto(NEW_CHAT_PATH);
+ const token = await getAccessToken(page);
+ const browserHeaders = { Authorization: `Bearer ${token}` };
+ const zip = new JSZip();
+ zip.file(
+ 'SKILL.md',
+ `---\nname: import-live-${Date.now()}\ndescription: Archive lifecycle test.\n---\n\nImported instructions.\n`,
+ );
+ zip.file('references/guide.txt', 'Imported guide bytes.');
+ const binary = Buffer.from([0, 255, 128, 1, 42]);
+ zip.file('assets/payload.bin', binary);
+ const imported = await request.post('/api/skills/import', {
+ headers: browserHeaders,
+ multipart: {
+ file: {
+ name: 'lifecycle.zip',
+ mimeType: 'application/zip',
+ buffer: await zip.generateAsync({ type: 'nodebuffer' }),
+ },
+ },
+ });
+ expect(imported.status(), await imported.text()).toBe(201);
+ const skill = await imported.json();
+ expect(skill._importSummary.filesFailed).toBe(0);
+ const rows = await mongoose.models.SkillFile.find({ skillId: skill._id }).lean();
+ expect(rows).toHaveLength(2);
+ for (const row of rows) expect(fs.existsSync(diskPath(String(row.filepath)))).toBe(true);
+ const textRead = await request.get(`/api/skills/${skill._id}/files/references/guide.txt`, {
+ headers: browserHeaders,
+ });
+ expect(textRead.status()).toBe(200);
+ expect((await textRead.json()).content).toBe('Imported guide bytes.');
+ const rawRead = await request.get(
+ `/api/skills/${skill._id}/files/assets/payload.bin?raw=true`,
+ { headers: browserHeaders },
+ );
+ expect(rawRead.status()).toBe(200);
+ expect(await rawRead.body()).toEqual(binary);
+ expect(
+ (await request.delete(`/api/skills/${skill._id}`, { headers: browserHeaders })).status(),
+ ).toBe(200);
+ for (const row of rows) expect(fs.existsSync(diskPath(String(row.filepath)))).toBe(false);
+ expect(await mongoose.models.SkillFile.countDocuments({ skillId: skill._id })).toBe(0);
+ });
+
+ test('ordinary local upload lists persisted metadata and physically deletes through the shared file route', async ({
+ page,
+ request,
+ }) => {
+ await page.goto(NEW_CHAT_PATH);
+ await selectMockEndpoint(page, MOCK_ENDPOINTS[1]);
+ const name = `ordinary-delete-${Date.now()}.csv`;
+ const uploaded = await uploadViaUnifiedButton(page, {
+ name,
+ mimeType: 'text/csv',
+ content: 'key,value\nhello,world\n',
+ });
+ expect(uploaded.ok()).toBe(true);
+ const file = await uploaded.json();
+ const token = await getAccessToken(page);
+ const browserHeaders = { Authorization: `Bearer ${token}` };
+ const listed = await request.get('/api/files', { headers: browserHeaders });
+ expect(listed.status()).toBe(200);
+ expect(await listed.json()).toContainEqual(expect.objectContaining({ file_id: file.file_id }));
+ expect(fs.existsSync(diskPath(file.filepath))).toBe(true);
+ const removed = await request.delete('/api/files', {
+ headers: browserHeaders,
+ data: { files: [file] },
+ });
+ expect(removed.status(), await removed.text()).toBe(200);
+ expect(fs.existsSync(diskPath(file.filepath))).toBe(false);
+ const after = await request.get('/api/files', { headers: browserHeaders });
+ expect(await after.json()).not.toContainEqual(
+ expect.objectContaining({ file_id: file.file_id }),
+ );
+ });
+
+ test('ordinary provider-routed upload reaches the model and persists in chat with an unambiguous attachment selector', async ({
+ page,
+ }) => {
+ await page.goto(NEW_CHAT_PATH);
+ await selectMockEndpoint(page, MOCK_ENDPOINTS[1]);
+ const name = `provider-lifecycle-${Date.now()}.md`;
+ const uploaded = await uploadViaUnifiedButton(page, {
+ name,
+ mimeType: 'text/markdown',
+ content: '# Provider attachment\n\nActual file bytes.\n',
+ });
+ expect(uploaded.ok()).toBe(true);
+ expect((await uploaded.json()).llmDeliveryPath).toBe('provider');
+ await expect(
+ page.getByTestId('composer-tray').getByRole('button', { name, exact: true }),
+ ).toBeVisible();
+ const input = page.getByRole('textbox', { name: 'Message input' });
+ await input.fill(`E2E_ASSERT_PROVIDER_FILE:${name}`);
+ const [stream] = await Promise.all([
+ page.waitForResponse(isAgentsStream),
+ page.getByTestId('send-button').click(),
+ ]);
+ expect(stream.ok()).toBe(true);
+ await expect(
+ page.getByTestId('messages-view').getByText(`E2E provider file assertion passed: ${name}`),
+ ).toBeVisible({ timeout: 20000 });
+ await expect(
+ page.getByTestId('messages-view').getByRole('button', { name, exact: true }),
+ ).toBeVisible();
+ await page.reload();
+ await expect(
+ page.getByTestId('messages-view').getByRole('button', { name, exact: true }),
+ ).toBeVisible();
+ });
+});
diff --git a/e2e/specs/mock/skill-management-delete.spec.ts b/e2e/specs/mock/skill-management-delete.spec.ts
new file mode 100644
index 0000000000..e2109a5208
--- /dev/null
+++ b/e2e/specs/mock/skill-management-delete.spec.ts
@@ -0,0 +1,191 @@
+import fs from 'fs';
+import path from 'path';
+import mongoose from 'mongoose';
+import { expect, test } from '@playwright/test';
+import { createMethods, createModels, tenantStorage } from '@librechat/data-schemas';
+import {
+ managementTenantId,
+ managementUserId,
+ startManagementOidc,
+} from '../../setup/agent-management';
+import { applyRuntimeEnv } from '../../setup/runtimeEnv';
+import { withMongo } from './db';
+
+const skillsPath = '/api/agents/v1/skills';
+const localBlobPath = (filepath: string) => {
+ const prefix = `/uploads/${managementUserId}/`;
+ expect(filepath.startsWith(prefix)).toBe(true);
+ return path.resolve('uploads', filepath.slice('/uploads/'.length));
+};
+
+test.describe('machine Skill management deletion', () => {
+ test.describe.configure({ mode: 'default' });
+ let oidc: Awaited<ReturnType<typeof startManagementOidc>>;
+ let headers: { Authorization: string };
+ let methods: ReturnType<typeof createMethods>;
+
+ test.beforeAll(async () => {
+ oidc = await startManagementOidc();
+ headers = { Authorization: `Bearer ${oidc.token}` };
+ applyRuntimeEnv();
+ await mongoose.connect(process.env.MONGO_URI!);
+ createModels(mongoose);
+ methods = createMethods(mongoose);
+ await tenantStorage.run({ tenantId: managementTenantId }, async () => {
+ await methods.initializeRoles();
+ await methods.seedDefaultRoles();
+ await methods.seedSystemGrants();
+ await mongoose.models.User.updateOne(
+ { _id: managementUserId },
+ {
+ $set: {
+ name: 'Skill Management E2E',
+ email: '[email protected]',
+ emailVerified: true,
+ provider: 'local',
+ role: 'ADMIN',
+ tenantId: managementTenantId,
+ },
+ },
+ { upsert: true },
+ );
+ });
+ });
+
+ test.afterAll(async () => {
+ try {
+ await withMongo(async (db) => {
+ for (const { name } of await db.listCollections({}, { nameOnly: true }).toArray()) {
+ if (!name.startsWith('system.')) {
+ await db.collection(name).deleteMany({ tenantId: managementTenantId });
+ }
+ }
+ });
+ await fs.promises.rm(path.resolve('uploads', managementUserId), {
+ recursive: true,
+ force: true,
+ });
+ } finally {
+ await mongoose.disconnect();
+ await oidc?.close();
+ }
+ });
+
+ test('creates, replaces, and deletes a Skill and its local blobs through HTTP', async ({
+ request,
+ }) => {
+ const withoutToken = await request.post(skillsPath, {
+ data: { name: 'unauthorized', description: 'Unauthorized.', body: 'No.' },
+ });
+ expect(withoutToken.status()).toBe(401);
+
+ const created = await request.post(skillsPath, {
+ headers,
+ data: {
+ name: `delete-e2e-${Date.now()}`,
+ description: 'Exercises machine Skill deletion with stored files.',
+ body: 'Test instructions.',
+ },
+ });
+ expect(created.status(), await created.text()).toBe(201);
+ const { id } = await created.json();
+ const filePath = `${skillsPath}/${id}/files/references/guide.md`;
+
+ const firstPut = await request.put(filePath, {
+ headers,
+ data: { content: 'First version.' },
+ });
+ expect(firstPut.status(), await firstPut.text()).toBe(200);
+ const firstFile = await tenantStorage.run({ tenantId: managementTenantId }, () =>
+ methods.getSkillFileByPath(id, 'references/guide.md'),
+ );
+ expect(firstFile?.source).toBe('local');
+ expect(firstFile?.filepath).toBeTruthy();
+ const oldBlobPath = localBlobPath(firstFile!.filepath);
+ expect(fs.existsSync(oldBlobPath)).toBe(true);
+
+ const secondPut = await request.put(filePath, {
+ headers,
+ data: { content: 'Second version.' },
+ });
+ expect(secondPut.status(), await secondPut.text()).toBe(200);
+ const secondFile = await tenantStorage.run({ tenantId: managementTenantId }, () =>
+ methods.getSkillFileByPath(id, 'references/guide.md'),
+ );
+ expect(secondFile?.filepath).not.toBe(firstFile?.filepath);
+ const newBlobPath = localBlobPath(secondFile!.filepath);
+ expect(fs.existsSync(newBlobPath)).toBe(true);
+ await expect.poll(() => fs.existsSync(oldBlobPath)).toBe(false);
+
+ const read = await request.get(filePath, { headers });
+ expect(read.status(), await read.text()).toBe(200);
+ expect((await read.json()).content).toBe('Second version.');
+
+ const deleted = await request.delete(`${skillsPath}/${id}`, { headers });
+ expect(deleted.status(), await deleted.text()).toBe(200);
+ expect(await deleted.json()).toEqual({ id, deleted: true, cleanupComplete: true });
+ expect(
+ await tenantStorage.run({ tenantId: managementTenantId }, () => methods.getSkillById(id)),
+ ).toBeNull();
+ expect(
+ await tenantStorage.run({ tenantId: managementTenantId }, () =>
+ methods.getSkillFileByPath(id, 'references/guide.md'),
+ ),
+ ).toBeNull();
+ expect(fs.existsSync(newBlobPath)).toBe(false);
+ expect((await request.get(`${skillsPath}/${id}`, { headers })).status()).toBe(404);
+ expect((await request.get(filePath, { headers })).status()).toBe(404);
+ });
+
+ test('retries whole-Skill cleanup after a transient local storage failure', async ({
+ request,
+ }) => {
+ const created = await request.post(skillsPath, {
+ headers,
+ data: {
+ name: `delete-retry-e2e-${Date.now()}`,
+ description: 'Exercises retry after local storage rejects deletion.',
+ body: 'Test retry instructions.',
+ },
+ });
+ expect(created.status(), await created.text()).toBe(201);
+ const { id } = await created.json();
+ const put = await request.put(`${skillsPath}/${id}/files/references/retry.md`, {
+ headers,
+ data: { content: 'Retry me.' },
+ });
+ expect(put.status(), await put.text()).toBe(200);
+ const file = await tenantStorage.run({ tenantId: managementTenantId }, () =>
+ methods.getSkillFileByPath(id, 'references/retry.md'),
+ );
+ expect(file?.filepath).toBeTruthy();
+ const blobPath = localBlobPath(file!.filepath);
+ expect(fs.existsSync(blobPath)).toBe(true);
+ const blobDirectory = path.dirname(blobPath);
+ const originalMode = fs.statSync(blobDirectory).mode & 0o777;
+
+ const deleted = await (async () => {
+ try {
+ await fs.promises.chmod(blobDirectory, 0o555);
+ return await request.delete(`${skillsPath}/${id}`, { headers });
+ } finally {
+ await fs.promises.chmod(blobDirectory, originalMode);
+ }
+ })();
+ expect(deleted.status(), await deleted.text()).toBe(200);
+ expect(await deleted.json()).toEqual({ id, deleted: true, cleanupComplete: false });
+ expect(fs.existsSync(blobPath)).toBe(true);
+
+ await expect
+ .poll(
+ async () => {
+ const skill = await tenantStorage.run({ tenantId: managementTenantId }, () =>
+ methods.getSkillById(id),
+ );
+ return { skillAbsent: skill == null, blobAbsent: !fs.existsSync(blobPath) };
+ },
+ { timeout: 15_000 },
+ )
+ .toEqual({ skillAbsent: true, blobAbsent: true });
+ });
+});
diff --git a/package.json b/package.json
index 1299e80829..468915b789 100644
--- a/package.json
+++ b/package.json
@@ -37,6 +37,7 @@
"reset-password": "node config/reset-password.js",
"ban-user": "node config/ban-user.js",
"delete-user": "node config/delete-user.js",
+ "recover-skill-upload": "npm run recover-skill-upload --workspace api --",
"reset-meili-sync": "node config/reset-meili-sync.js",
"create-error-convo": "node config/create-error-convo.js",
"update-banner": "node config/update-banner.js",
diff --git a/packages/api/src/agents/handlers.spec.ts b/packages/api/src/agents/handlers.spec.ts
index e6e0e83b25..c7baabe333 100644
--- a/packages/api/src/agents/handlers.spec.ts
+++ b/packages/api/src/agents/handlers.spec.ts
@@ -4630,6 +4630,46 @@ describe('createToolExecuteHandler', () => {
);
});
+ it('reports a safe error when Skill deletion wins the agent file-write race', async () => {
+ const saveSkillFileContent = jest.fn(async () => {
+ throw Object.assign(new Error('Skill is being deleted'), { code: 'SKILL_DELETING' });
+ });
+ const handler = makeAuthoringHandler({
+ getSkillByName: jest.fn(async () => ({
+ _id: SKILL_ID,
+ name: 'edit-skill',
+ body: '# Existing',
+ fileCount: 1,
+ version: 1,
+ })),
+ getSkillFileByPath: jest.fn(async () => ({
+ file_id: 'revision-1',
+ content: 'hello old\n',
+ isBinary: false,
+ mimeType: 'text/markdown',
+ bytes: 10,
+ filepath: '/tmp/a.md',
+ source: 'local',
+ relativePath: 'references/a.md',
+ })),
+ saveSkillFileContent,
+ });
+ const [result] = await invokeHandler(handler, [
+ {
+ id: 'call_deleting',
+ name: 'edit_file',
+ args: {
+ path: 'skills/edit-skill/references/a.md',
+ old_text: 'hello old',
+ new_text: 'hello new',
+ },
+ },
+ ]);
+ expect(result.status).toBe('error');
+ expect(result.errorMessage).toContain('Skill is being deleted');
+ expect(result.errorMessage).not.toContain('Internal');
+ });
+
it.each(['github', 'notion'] as const)(
'does not allow an agent to modify a %s-managed skill file',
async (source) => {
diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts
index 1627d769b8..75807fdcb9 100644
--- a/packages/api/src/agents/handlers.ts
+++ b/packages/api/src/agents/handlers.ts
@@ -3837,6 +3837,9 @@ async function writeBundledSkillFile({
: `Skill file changed while editing. Re-read ${displayPath} and retry.`,
);
}
+ if (error instanceof Error && 'code' in error && error.code === 'SKILL_DELETING') {
+ return errorResult(tc, `Skill is being deleted. Re-read ${displayPath} before retrying.`);
+ }
throw error;
}
const action = created ? 'Created' : 'Updated';
diff --git a/packages/api/src/skills/__tests__/import.test.ts b/packages/api/src/skills/__tests__/import.test.ts
index 33eeb7b6c1..ce1a2943e3 100644
--- a/packages/api/src/skills/__tests__/import.test.ts
+++ b/packages/api/src/skills/__tests__/import.test.ts
@@ -71,6 +71,19 @@ function mockImportDeps(limits?: ImportSkillDeps['limits']): ImportSkillDeps {
failedCleanupSteps: [],
})),
upsertSkillFile: jest.fn(async () => skillFile),
+ getSkillFileByPath: jest.fn(async () => null),
+ reserveBlob: jest.fn(async (_req, { userId, fileName, tenantId }) => ({
+ filepath: '/tmp/imported-file',
+ source: 'local',
+ writeLocation: { userId, fileName, basePath: 'uploads' as const, tenantId },
+ })),
+ reserveSkillUploadIntent: jest.fn(async (input) => ({
+ ...input,
+ skillId: new Types.ObjectId(input.skillId),
+ author: new Types.ObjectId(input.author),
+ state: 'uploading' as const,
+ })),
+ transitionSkillUploadIntent: jest.fn(async () => true),
saveBuffer: jest.fn(async () => ({ filepath: '/tmp/imported-file', source: 'local' })),
deleteFile: jest.fn(async () => undefined),
grantPermission: jest.fn(async () => undefined),
@@ -601,7 +614,7 @@ describe('createImportHandler', () => {
);
});
- it('retries an orphan blob after SkillFile cleanup remains incomplete', async () => {
+ it('retains an orphan intent after immediate cleanup fails', async () => {
const deps = mockImportDeps();
deps.deleteSkill = jest.fn(async () => ({
deleted: true,
@@ -624,7 +637,7 @@ describe('createImportHandler', () => {
await handler(mockZipRequest(await zipWithAdditionalFiles(1, 64)), res);
expect(deps.deleteSkill).toHaveBeenCalledTimes(2);
- expect(deps.deleteFile).toHaveBeenCalledTimes(2);
+ expect(deps.deleteFile).toHaveBeenCalledTimes(1);
expect(res.status).toHaveBeenCalledWith(500);
expect(importFailure(res.body).error).toBe('skill_import_cleanup_incomplete');
});
@@ -779,6 +792,12 @@ describe('createImportHandler', () => {
it('rolls back the skill and its stored blobs when one archive file fails', async () => {
const deps = mockImportDeps();
let savedFiles = 0;
+ let reservedFiles = 0;
+ deps.reserveBlob = jest.fn(async (_req, { userId, fileName, tenantId }) => ({
+ filepath: `/tmp/imported-${reservedFiles++}`,
+ source: 'local',
+ writeLocation: { userId, fileName, basePath: 'uploads' as const, tenantId },
+ }));
deps.saveBuffer = jest.fn(async () => ({
filepath: `/tmp/imported-${savedFiles++}`,
source: 'local',
@@ -812,6 +831,47 @@ describe('createImportHandler', () => {
expect(res.body).not.toHaveProperty('_importSummary');
});
+ it('does not delete persisted blobs twice when durable skill deletion owns storage cleanup', async () => {
+ const deps = mockImportDeps();
+ let savedFiles = 0;
+ let reservedFiles = 0;
+ deps.reserveBlob = jest.fn(async (_req, { userId, fileName, tenantId }) => ({
+ filepath: `/tmp/imported-${reservedFiles++}`,
+ source: 'local',
+ writeLocation: { userId, fileName, basePath: 'uploads' as const, tenantId },
+ }));
+ deps.saveBuffer = jest.fn(async () => ({
+ filepath: `/tmp/imported-${savedFiles++}`,
+ source: 'local',
+ })) as ImportSkillDeps['saveBuffer'];
+ deps.upsertSkillFile = jest
+ .fn()
+ .mockResolvedValueOnce({ _id: new Types.ObjectId() })
+ .mockRejectedValueOnce(
+ new Error('write conflict'),
+ ) as unknown as ImportSkillDeps['upsertSkillFile'];
+ deps.deleteSkill = jest.fn(async () => ({
+ deleted: true,
+ skillAbsent: true,
+ cleanupComplete: true,
+ failedCleanupSteps: [],
+ blobCleanupManaged: true,
+ }));
+ deps.deleteFile = jest.fn(async () => undefined);
+ const handler = createImportHandler(deps);
+ const res = mockResponse();
+
+ await handler(mockZipRequest(await zipWithAdditionalFiles(2, 64)), res);
+
+ expect(deps.deleteSkill).toHaveBeenCalledTimes(1);
+ // The failed, rowless upload is cleaned inline; the durable worker owns the first file.
+ expect(deps.deleteFile).toHaveBeenCalledTimes(1);
+ expect(deps.deleteFile).not.toHaveBeenCalledWith(
+ expect.anything(),
+ expect.objectContaining({ filepath: '/tmp/imported-0' }),
+ );
+ });
+
it('rolls back an archive whose entry path is unsafe instead of importing it partially', async () => {
const deps = mockImportDeps();
deps.deleteFile = jest.fn(async () => undefined);
diff --git a/packages/api/src/skills/deletion.spec.ts b/packages/api/src/skills/deletion.spec.ts
index f607de0b72..23b20e3b00 100644
--- a/packages/api/src/skills/deletion.spec.ts
+++ b/packages/api/src/skills/deletion.spec.ts
@@ -1,4 +1,8 @@
-import type { SkillDeletionRecord, SkillUploadIntentRecord } from '@librechat/data-schemas';
+import type {
+ SkillDeletionFileRecord,
+ SkillDeletionRecord,
+ SkillUploadIntentRecord,
+} from '@librechat/data-schemas';
import type { SkillDeletionDeps } from './deletion';
import { runSkillDeletion, SkillDeletionClaimLostError } from './deletion';
@@ -8,6 +12,13 @@ const journal = {
skillId,
} satisfies SkillDeletionRecord;
+async function* batches<Row>(rows: Row[], size: number): AsyncGenerator<Row[]> {
+ const snapshot = [...rows];
+ for (let offset = 0; offset < snapshot.length; offset += size) {
+ yield snapshot.slice(offset, offset + size);
+ }
+}
+
function harness() {
const files = [
{
@@ -16,13 +27,24 @@ function harness() {
source: 'local',
author,
},
- ] satisfies Awaited<ReturnType<SkillDeletionDeps['listSkillFiles']>>;
+ ] satisfies SkillDeletionFileRecord[];
const intents: SkillUploadIntentRecord[] = [];
const deps: SkillDeletionDeps = {
beginSkillDeletion: jest.fn(async () => journal),
getSkillDeletion: jest.fn(async () => journal),
- listSkillFiles: jest.fn(async () => files),
- listSkillUploadIntents: jest.fn(async () => intents),
+ iterateSkillCleanupFiles: jest.fn((_id, size) => batches(files, size)),
+ iterateSkillCleanupIntents: jest.fn((_id, size) =>
+ batches(
+ intents.filter((intent) => intent.state !== 'committed' || !intent.cleanedAt),
+ size,
+ ),
+ ),
+ hasInFlightSkillUploads: jest.fn(async () =>
+ intents.some((intent) => intent.state === 'uploading' || intent.state === 'committing'),
+ ),
+ listExistingSkillFileIds: jest.fn(async (_id, ids) =>
+ files.filter((file) => ids.includes(file.file_id)).map((file) => file.file_id),
+ ),
deleteBlob: jest.fn(async () => undefined),
deleteReservedBlob: jest.fn(async () => undefined),
deleteSkillFileAfterBlob: jest.fn(async (_id, fileId) => {
@@ -298,3 +320,138 @@ it('bounds independent file cleanup while deleting each row after its blob', asy
expect(h.deps.deleteSkillFileAfterBlob).toHaveBeenCalledTimes(3);
expect(h.files).toHaveLength(0);
});
+
+it('fully settles each file page before requesting the next one', async () => {
+ const h = harness();
+ const template = h.files[0];
+ h.files.splice(
+ 0,
+ 1,
+ ...Array.from({ length: 11 }, (_, index) => ({ ...template, file_id: `page-file-${index}` })),
+ );
+ const closed = jest.fn();
+ const sizes: number[] = [];
+ const original = h.deps.iterateSkillCleanupFiles;
+ h.deps.iterateSkillCleanupFiles = async function* (id, size) {
+ let processed = 0;
+ try {
+ for await (const page of original(id, size)) {
+ expect(h.deps.deleteSkillFileAfterBlob).toHaveBeenCalledTimes(processed);
+ sizes.push(page.length);
+ yield page;
+ processed += page.length;
+ }
+ } finally {
+ closed();
+ }
+ };
+ await expect(
+ runSkillDeletion(skillId, h.deps, 2, undefined, undefined, 3),
+ ).resolves.toMatchObject({ cleanupComplete: true });
+ expect(sizes).toEqual([3, 3, 3, 2]);
+ expect(h.files).toHaveLength(0);
+ expect(closed).toHaveBeenCalledTimes(1);
+});
+
+it('uses one bounded file lookup per intent page and retries a failed later page', async () => {
+ const h = harness();
+ h.files.length = 0;
+ for (let index = 0; index < 11; index++) {
+ h.intents.push({
+ skillId,
+ author,
+ fileId: `intent-${index}`,
+ source: 'local',
+ storageKey: `/uploads/${index}`,
+ state: 'committed',
+ });
+ }
+ (h.deps.deleteReservedBlob as jest.Mock).mockImplementation(
+ async (intent: SkillUploadIntentRecord) => {
+ if (intent.fileId === 'intent-7') throw new Error('temporary provider failure');
+ },
+ );
+ await expect(
+ runSkillDeletion(skillId, h.deps, 2, undefined, undefined, 3),
+ ).resolves.toMatchObject({ cleanupComplete: false, failedSteps: ['upload_intents'] });
+ expect(h.intents.filter((intent) => !intent.cleanedAt).map((intent) => intent.fileId)).toEqual([
+ 'intent-7',
+ ]);
+ expect(h.deps.listExistingSkillFileIds).toHaveBeenCalledTimes(4);
+ for (const [, ids] of (h.deps.listExistingSkillFileIds as jest.Mock).mock.calls)
+ expect(ids.length).toBeLessThanOrEqual(3);
+ (h.deps.deleteReservedBlob as jest.Mock).mockResolvedValue(undefined);
+ await expect(
+ runSkillDeletion(skillId, h.deps, 2, undefined, undefined, 3),
+ ).resolves.toMatchObject({ cleanupComplete: true });
+ expect(h.deps.deleteReservedBlob).toHaveBeenCalledTimes(12);
+});
+
+it('checks all in-flight writers before opening either cleanup cursor', async () => {
+ const h = harness();
+ for (let index = 0; index < 20; index++) {
+ h.intents.push({
+ skillId,
+ author,
+ fileId: `intent-${index}`,
+ source: 'local',
+ storageKey: `/uploads/${index}`,
+ state: index === 19 ? 'uploading' : 'committed',
+ });
+ }
+ await expect(
+ runSkillDeletion(skillId, h.deps, 2, undefined, undefined, 3),
+ ).resolves.toMatchObject({ cleanupComplete: false });
+ expect(h.deps.iterateSkillCleanupFiles).not.toHaveBeenCalled();
+ expect(h.deps.iterateSkillCleanupIntents).not.toHaveBeenCalled();
+ expect(h.deps.deleteBlob).not.toHaveBeenCalled();
+});
+
+it('closes the current page iterator and does not fetch later pages after claim loss', async () => {
+ const h = harness();
+ const closed = jest.fn();
+ const nextPage = jest.fn();
+ h.deps.iterateSkillCleanupFiles = async function* () {
+ try {
+ yield [...h.files];
+ nextPage();
+ yield [...h.files];
+ } finally {
+ closed();
+ }
+ };
+ let lost = false;
+ (h.deps.deleteBlob as jest.Mock).mockImplementation(async () => {
+ lost = true;
+ });
+ await expect(
+ runSkillDeletion(
+ skillId,
+ h.deps,
+ 1,
+ async () => {
+ if (lost) throw new SkillDeletionClaimLostError();
+ },
+ undefined,
+ 1,
+ ),
+ ).rejects.toBeInstanceOf(SkillDeletionClaimLostError);
+ expect(closed).toHaveBeenCalledTimes(1);
+ expect(nextPage).not.toHaveBeenCalled();
+ expect(h.files).toHaveLength(1);
+});
+
+it('bounds repeated failure bookkeeping across pages', async () => {
+ const h = harness();
+ const template = h.files[0];
+ h.files.splice(
+ 0,
+ 1,
+ ...Array.from({ length: 30 }, (_, index) => ({ ...template, file_id: `failed-${index}` })),
+ );
+ (h.deps.deleteBlob as jest.Mock).mockRejectedValue(new Error('provider unavailable'));
+ await expect(
+ runSkillDeletion(skillId, h.deps, 2, undefined, undefined, 3),
+ ).resolves.toMatchObject({ failedSteps: ['skill_files'] });
+ expect(h.files).toHaveLength(30);
+});
diff --git a/packages/api/src/skills/deletion.ts b/packages/api/src/skills/deletion.ts
index cf237a2a69..d619760e61 100644
--- a/packages/api/src/skills/deletion.ts
+++ b/packages/api/src/skills/deletion.ts
@@ -1,4 +1,7 @@
-import { DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY } from 'librechat-data-provider';
+import {
+ DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY,
+ DEFAULT_SKILL_DELETION_BATCH_SIZE,
+} from 'librechat-data-provider';
import type {
DeleteSkillCleanupStep,
DeleteSkillResult,
@@ -11,8 +14,16 @@ import { createConcurrencyLimiter } from '~/utils/promise';
export interface SkillDeletionDeps {
beginSkillDeletion: (id: string) => Promise<SkillDeletionRecord | null>;
getSkillDeletion: (id: string) => Promise<SkillDeletionRecord | null>;
- listSkillFiles: (id: string) => Promise<SkillDeletionFileRecord[]>;
- listSkillUploadIntents: (id: string) => Promise<SkillUploadIntentRecord[]>;
+ iterateSkillCleanupFiles: (
+ id: string,
+ batchSize: number,
+ ) => AsyncIterable<SkillDeletionFileRecord[]>;
+ iterateSkillCleanupIntents: (
+ id: string,
+ batchSize: number,
+ ) => AsyncIterable<SkillUploadIntentRecord[]>;
+ hasInFlightSkillUploads: (id: string) => Promise<boolean>;
+ listExistingSkillFileIds: (id: string, fileIds: string[]) => Promise<string[]>;
deleteBlob: (file: SkillDeletionFileRecord) => Promise<void>;
deleteReservedBlob: (intent: SkillUploadIntentRecord) => Promise<void>;
deleteSkillFileAfterBlob: (skillId: string, fileId: string) => Promise<boolean>;
@@ -47,7 +58,10 @@ export async function runSkillDeletion(
fileCleanupConcurrency: number = DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY,
assertClaim: () => Promise<void> = async () => undefined,
loadedJournal?: SkillDeletionRecord,
+ manifestBatchSize: number = DEFAULT_SKILL_DELETION_BATCH_SIZE,
): Promise<SkillDeletionProgress> {
+ if (!Number.isInteger(manifestBatchSize) || manifestBatchSize < 1)
+ throw new Error('Invalid cleanup batch size');
await assertClaim();
const journal =
loadedJournal ?? (await deps.beginSkillDeletion(id)) ?? (await deps.getSkillDeletion(id));
@@ -60,27 +74,21 @@ export async function runSkillDeletion(
failedSteps: [],
};
}
- const failedSteps: DeleteSkillCleanupStep[] = [];
- const [intents, files] = await Promise.all([
- deps.listSkillUploadIntents(journal.skillId),
- deps.listSkillFiles(id),
- ]);
- const requiresFullCleanup =
- files.length > 0 ||
- intents.some(
- (intent) =>
- intent.state === 'uploading' ||
- intent.state === 'committing' ||
- (intent.state === 'committed' && !intent.cleanedAt),
- );
- if (journal.completedAt && requiresFullCleanup) {
- await assertClaim();
- await deps.reopenSkillDeletion(journal.skillId);
- }
+ const failures = new Set<DeleteSkillCleanupStep>();
+ let requiresFullCleanup = false;
+ const noteLiveWork = async () => {
+ if (requiresFullCleanup) return;
+ if (journal.completedAt) {
+ await assertClaim();
+ await deps.reopenSkillDeletion(journal.skillId);
+ }
+ requiresFullCleanup = true;
+ };
// A writer may still finish uploading or committing after this scan. An
// operator can inspect and recover an intent with `recover-skill-upload`
// after draining writers; a timer must not infer that a slow upload is dead.
- if (intents.some((intent) => intent.state === 'uploading' || intent.state === 'committing')) {
+ if (await deps.hasInFlightSkillUploads(journal.skillId)) {
+ await noteLiveWork();
return {
found: true,
deleted: false,
@@ -89,79 +97,104 @@ export async function runSkillDeletion(
failedSteps: ['upload_intents'],
};
}
- const currentFileIds = new Set(files.map((file) => file.file_id));
- const cleanedFileIds = new Set<string>();
const limit = createConcurrencyLimiter(fileCleanupConcurrency);
- const fileResults = await Promise.allSettled(
- files.map((file) =>
- limit(async () => {
- await assertClaim();
- try {
- await deps.deleteBlob(file);
- } catch {
- failedSteps.push('skill_files');
- return;
- }
- await assertClaim();
- try {
- await deps.deleteSkillFileAfterBlob(id, file.file_id);
- cleanedFileIds.add(file.file_id);
- } catch {
- failedSteps.push('skill_files');
- }
- }),
- ),
- );
- const rejectedFile = fileResults.find((result) => result.status === 'rejected');
- if (rejectedFile?.status === 'rejected') throw rejectedFile.reason;
-
- const intentResults = await Promise.allSettled(
- intents.map((intent) =>
- limit(async () => {
- if (intent.state === 'committed') {
- if (intent.cleanedAt) return;
- if (currentFileIds.has(intent.fileId) && !cleanedFileIds.has(intent.fileId)) {
- // Keep the intent with its current row for the next retry.
+ for await (const files of deps.iterateSkillCleanupFiles(id, manifestBatchSize)) {
+ if (files.length === 0) continue;
+ await noteLiveWork();
+ const fileResults = await Promise.allSettled(
+ files.map((file) =>
+ limit(async () => {
+ await assertClaim();
+ try {
+ await deps.deleteBlob(file);
+ } catch {
+ failures.add('skill_files');
+ return;
+ }
+ await assertClaim();
+ try {
+ await deps.deleteSkillFileAfterBlob(id, file.file_id);
+ } catch {
+ failures.add('skill_files');
return;
}
await assertClaim();
try {
- if (!cleanedFileIds.has(intent.fileId)) {
+ // Persist this progress instead of retaining every cleaned file ID in memory.
+ await deps.markSkillUploadIntentCleaned(file.file_id);
+ } catch {
+ failures.add('upload_intents');
+ }
+ }),
+ ),
+ );
+ const rejectedFile = fileResults.find((result) => result.status === 'rejected');
+ if (rejectedFile?.status === 'rejected') throw rejectedFile.reason;
+ }
+
+ for await (const intents of deps.iterateSkillCleanupIntents(journal.skillId, manifestBatchSize)) {
+ const committedIds: string[] = [];
+ for (const intent of intents) {
+ if (intent.state === 'uploading' || intent.state === 'committing') {
+ await noteLiveWork();
+ return {
+ found: true,
+ deleted: false,
+ skillAbsent: journal.completedAt != null,
+ cleanupComplete: false,
+ failedSteps: ['upload_intents'],
+ };
+ }
+ if (intent.state === 'committed' && !intent.cleanedAt) committedIds.push(intent.fileId);
+ }
+ if (committedIds.length > 0) await noteLiveWork();
+ const currentFileIds = new Set(await deps.listExistingSkillFileIds(id, committedIds));
+ const intentResults = await Promise.allSettled(
+ intents.map((intent) =>
+ limit(async () => {
+ if (intent.state === 'committed') {
+ if (intent.cleanedAt) return;
+ if (currentFileIds.has(intent.fileId)) {
+ // Keep the intent with its current row for the next retry.
+ return;
+ }
+ await assertClaim();
+ try {
await deps.deleteReservedBlob(intent);
+ } catch {
+ failures.add('upload_intents');
+ return;
+ }
+ await assertClaim();
+ try {
+ await deps.markSkillUploadIntentCleaned(intent.fileId);
+ } catch {
+ failures.add('upload_intents');
}
- } catch {
- failedSteps.push('upload_intents');
return;
}
await assertClaim();
try {
- await deps.markSkillUploadIntentCleaned(intent.fileId);
+ await deps.deleteReservedBlob(intent);
+ // Keep abandoned intents as tombstones: a zombie upload can finish
+ // after this cleanup and must be discovered on the next sweep.
} catch {
- failedSteps.push('upload_intents');
+ failures.add('upload_intents');
}
- return;
- }
- await assertClaim();
- try {
- await deps.deleteReservedBlob(intent);
- // Keep abandoned intents as tombstones: a zombie upload can finish
- // after this cleanup and must be discovered on the next sweep.
- } catch {
- failedSteps.push('upload_intents');
- }
- }),
- ),
- );
- const rejectedIntent = intentResults.find((result) => result.status === 'rejected');
- if (rejectedIntent?.status === 'rejected') throw rejectedIntent.reason;
+ }),
+ ),
+ );
+ const rejectedIntent = intentResults.find((result) => result.status === 'rejected');
+ if (rejectedIntent?.status === 'rejected') throw rejectedIntent.reason;
+ }
- if (failedSteps.length > 0) {
+ if (failures.size > 0) {
return {
found: true,
deleted: false,
skillAbsent: journal.completedAt != null,
cleanupComplete: false,
- failedSteps,
+ failedSteps: [...failures],
};
}
if (journal.completedAt && !requiresFullCleanup) {
diff --git a/packages/api/src/skills/deletionService.restart.spec.ts b/packages/api/src/skills/deletionService.restart.spec.ts
new file mode 100644
index 0000000000..d6006ebb6f
--- /dev/null
+++ b/packages/api/src/skills/deletionService.restart.spec.ts
@@ -0,0 +1,200 @@
+import mongoose, { Types } from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import {
+ createMethods,
+ createModels,
+ createSkillDeletionPersistenceAdapter,
+ tenantStorage,
+} from '@librechat/data-schemas';
+import type { AppConfig } from '@librechat/data-schemas';
+import { createSkillDeletionService } from './deletionService';
+
+describe('Skill cleanup across a worker restart', () => {
+ let mongo: MongoMemoryServer;
+
+ beforeAll(async () => {
+ mongo = await MongoMemoryServer.create();
+ await mongoose.connect(mongo.getUri());
+ createModels(mongoose);
+ });
+
+ afterAll(async () => {
+ await mongoose.disconnect();
+ await mongo.stop();
+ });
+
+ it('finishes a superseded committed upload without reporting successful cleanup as a lost claim', async () => {
+ const db = createMethods(mongoose);
+ const author = new Types.ObjectId();
+ const { skill } = await db.createSkill({
+ name: 'completed-upload',
+ description: 'Exercise cleanup completion with real persistence.',
+ body: 'Instructions.',
+ author,
+ authorName: 'Test',
+ });
+ const fileId = 'completed-upload-file';
+ await mongoose.models.SkillUploadIntent.create({
+ skillId: skill._id,
+ fileId,
+ source: 's3',
+ storageKey: `uploads/${author}/old.txt`,
+ author,
+ writerId: 'finished-writer',
+ state: 'committed',
+ cleanupRequestedAt: new Date(),
+ });
+ const deleteFile = jest.fn(async () => undefined);
+ const logger = { error: jest.fn(), warn: jest.fn() };
+ const service = createSkillDeletionService({
+ persistence: { ...db, ...createSkillDeletionPersistenceAdapter(db) },
+ loadConfig: async () => undefined,
+ getStrategyFunctions: () => ({ deleteFile }),
+ runAsSystem: async (callback) => callback(),
+ logger,
+ });
+ const sweep = service.startSkillDeletionSweep();
+ try {
+ for (let attempt = 0; attempt < 100; attempt++) {
+ const intent = await db.getSkillUploadIntent(fileId);
+ if (intent?.lastCleanupAttemptAt && !intent.claimOwner) break;
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ }
+ const intent = await db.getSkillUploadIntent(fileId);
+ expect(intent?.cleanedAt).toBeInstanceOf(Date);
+ expect(intent?.lastCleanupAttemptAt).toBeInstanceOf(Date);
+ expect(intent?.claimOwner).toBeUndefined();
+ expect(intent?.nextCheckAt).toBeInstanceOf(Date);
+ expect(await db.listAbandonedSkillUploadIntents()).toEqual([]);
+ expect(deleteFile).toHaveBeenCalledTimes(1);
+ expect(logger.error).not.toHaveBeenCalled();
+ } finally {
+ sweep.stop();
+ }
+ });
+
+ it('retains a failed S3 deletion and retries it with a new worker instance', async () => {
+ const db = createMethods(mongoose);
+ const author = new Types.ObjectId();
+ const tenantId = 'restart-test';
+ const { skill } = await tenantStorage.run({ tenantId }, () =>
+ db.createSkill({
+ name: 'restart-test',
+ description: 'Exercise durable Skill deletion after a worker restart.',
+ body: 'Test instructions.',
+ author,
+ authorName: 'Test',
+ tenantId,
+ }),
+ );
+ const fileId = 'restart-file';
+ const storageKey = 'r/us-east-2/uploads/restart-test/guide.md';
+ await tenantStorage.run({ tenantId }, () =>
+ db.upsertSkillFile({
+ skillId: skill._id,
+ relativePath: 'references/guide.md',
+ file_id: fileId,
+ filename: 'guide.md',
+ filepath: 'https://storage.example.test/restart-test/guide.md',
+ storageKey,
+ storageRegion: 'us-east-2',
+ source: 's3',
+ mimeType: 'text/markdown',
+ bytes: 10,
+ author,
+ tenantId,
+ }),
+ );
+
+ let storageAvailable = false;
+ const deleteFile = jest.fn(async () => {
+ if (!storageAvailable) throw new Error('temporary S3 outage');
+ });
+ const makeWorker = () =>
+ createSkillDeletionService({
+ persistence: { ...db, ...createSkillDeletionPersistenceAdapter(db) },
+ loadConfig: async () =>
+ ({ skillDeletion: { finalizationEnabled: true } }) as unknown as AppConfig,
+ getStrategyFunctions: (source) => (source === 's3' ? { deleteFile } : {}),
+ runAsSystem: async (callback) => callback(),
+ logger: { error: jest.fn(), warn: jest.fn() },
+ });
+
+ const firstWorker = makeWorker();
+ const pending = await firstWorker.deleteSkillById(skill._id.toString());
+ expect(pending.cleanupComplete).toBe(false);
+ expect(await db.getSkillFileByPath(skill._id, 'references/guide.md')).not.toBeNull();
+ expect(await db.getSkillDeletion(skill._id.toString())).not.toBeNull();
+
+ // Simulate a fresh process using the same persisted database and provider.
+ storageAvailable = true;
+ const restartedWorker = makeWorker();
+ const sweep = restartedWorker.startSkillDeletionSweep({ sweepIntervalMs: 1_000 });
+ try {
+ for (let attempt = 0; attempt < 50; attempt++) {
+ if ((await mongoose.models.Skill.findById(skill._id)) == null) break;
+ await new Promise((resolve) => setTimeout(resolve, 100));
+ }
+ expect(await mongoose.models.Skill.findById(skill._id)).toBeNull();
+ expect(await db.getSkillFileByPath(skill._id, 'references/guide.md')).toBeNull();
+ expect(deleteFile).toHaveBeenCalledWith(
+ expect.objectContaining({ user: expect.objectContaining({ tenantId }) }),
+ expect.objectContaining({ storageKey, storageRegion: 'us-east-2', tenantId }),
+ );
+ } finally {
+ sweep.stop();
+ }
+ });
+
+ it('cleans a multi-page manifest and retries only the failed file with real persistence', async () => {
+ const db = createMethods(mongoose);
+ const author = new Types.ObjectId();
+ const { skill } = await db.createSkill({
+ name: 'paged-cleanup',
+ description: 'Paged cleanup retry.',
+ body: '',
+ author,
+ authorName: 'Test',
+ });
+ await mongoose.models.SkillFile.insertMany(
+ Array.from({ length: 37 }, (_, index) => ({
+ skillId: skill._id,
+ file_id: `paged-retry-${index}`,
+ relativePath: `${index}.md`,
+ filename: `${index}.md`,
+ filepath: `uploads/${author}/${index}`,
+ source: 's3',
+ mimeType: 'text/markdown',
+ bytes: 4,
+ author,
+ })),
+ );
+ let available = false;
+ const deleted = new Map<string, number>();
+ const deleteFile = jest.fn(async (_request, file: { file_id?: string; filepath: string }) => {
+ const key = file.filepath;
+ if (!available && key.endsWith('/23')) throw new Error('temporary provider failure');
+ deleted.set(key, (deleted.get(key) ?? 0) + 1);
+ });
+ const makeService = () =>
+ createSkillDeletionService({
+ persistence: { ...db, ...createSkillDeletionPersistenceAdapter(db) },
+ loadConfig: async () =>
+ ({ skillDeletion: { finalizationEnabled: true, batchSize: 5 } }) as unknown as AppConfig,
+ getStrategyFunctions: () => ({ deleteFile }),
+ runAsSystem: async (callback) => callback(),
+ logger: { error: jest.fn(), warn: jest.fn() },
+ });
+ const first = await makeService().deleteSkillById(skill._id.toString());
+ expect(first.cleanupComplete).toBe(false);
+ expect(await mongoose.models.SkillFile.countDocuments({ skillId: skill._id })).toBe(1);
+ expect(deleted.size).toBe(36);
+ available = true;
+ expect((await makeService().deleteSkillById(skill._id.toString())).cleanupComplete).toBe(true);
+ expect(await mongoose.models.SkillFile.countDocuments({ skillId: skill._id })).toBe(0);
+ expect(await mongoose.models.Skill.findById(skill._id)).toBeNull();
+ expect(deleted.size).toBe(37);
+ expect([...deleted.values()].every((count) => count === 1)).toBe(true);
+ expect(deleteFile).toHaveBeenCalledTimes(38);
+ });
+});
diff --git a/packages/api/src/skills/deletionService.spec.ts b/packages/api/src/skills/deletionService.spec.ts
index 3c271df4a9..13b82dd6a6 100644
--- a/packages/api/src/skills/deletionService.spec.ts
+++ b/packages/api/src/skills/deletionService.spec.ts
@@ -23,10 +23,12 @@ it('records failed cleanup attempts before releasing the claim', async () => {
}),
beginSkillDeletion: jest.fn(async () => journal),
getSkillDeletion: jest.fn(async (): Promise<typeof journal | null> => journal),
- listSkillFiles: jest.fn(async () => {
- throw new Error('storage unavailable');
+ iterateSkillCleanupFiles: jest.fn(async function* () {
+ yield await Promise.reject(new Error('storage unavailable'));
}),
- listSkillUploadIntents: jest.fn(async () => []),
+ iterateSkillCleanupIntents: jest.fn(async function* () {}),
+ hasInFlightSkillUploads: jest.fn(async () => false),
+ listExistingSkillFileIds: jest.fn(async () => []),
deleteSkillFileAfterBlob: jest.fn(async () => true),
finalizeSkillDeletion: jest.fn(async () => ({
deleted: true,
@@ -66,7 +68,7 @@ it('records failed cleanup attempts before releasing the claim', async () => {
});
it.each([false, true])(
- 'reconciles an abandoned intent on an active Skill while preserving a live file row: row=%s',
+ 'reconciles an old intent on an active Skill while preserving a live file row: row=%s',
async (rowExists) => {
const skillId = '507f1f77bcf86cd799439011';
const author = '507f1f77bcf86cd799439012';
@@ -76,7 +78,7 @@ it.each([false, true])(
source: 'local',
storageKey: '/uploads/abandoned-file',
author,
- state: 'abandoned' as const,
+ state: rowExists ? ('committed' as const) : ('abandoned' as const),
};
let released: () => void = () => undefined;
const releaseObserved = new Promise<void>((resolve) => {
@@ -92,8 +94,10 @@ it.each([false, true])(
releaseSkillDeletionClaim: jest.fn(async () => undefined),
beginSkillDeletion: jest.fn(async () => null),
getSkillDeletion: jest.fn(async () => null),
- listSkillFiles: jest.fn(async () => []),
- listSkillUploadIntents: jest.fn(async () => []),
+ iterateSkillCleanupFiles: jest.fn(async function* () {}),
+ iterateSkillCleanupIntents: jest.fn(async function* () {}),
+ hasInFlightSkillUploads: jest.fn(async () => false),
+ listExistingSkillFileIds: jest.fn(async () => []),
deleteSkillFileAfterBlob: jest.fn(async () => true),
finalizeSkillDeletion: jest.fn(async () => ({
deleted: false,
@@ -163,8 +167,12 @@ function claimedServiceHarness() {
scheduleSkillDeletionRecheck: jest.fn(async () => true),
beginSkillDeletion: jest.fn(async () => journal),
getSkillDeletion: jest.fn(async (): Promise<typeof journal | null> => journal),
- listSkillFiles: jest.fn(async () => files),
- listSkillUploadIntents: jest.fn(async () => []),
+ iterateSkillCleanupFiles: jest.fn(async function* () {
+ yield [...files];
+ }),
+ iterateSkillCleanupIntents: jest.fn(async function* () {}),
+ hasInFlightSkillUploads: jest.fn(async () => false),
+ listExistingSkillFileIds: jest.fn(async () => []),
deleteSkillFileAfterBlob: jest.fn(async () => {
files.length = 0;
return true;
@@ -205,6 +213,38 @@ function claimedServiceHarness() {
return { skillId, persistence, deleteFile, loadConfig, logger, service };
}
+it('retries a superseded committed blob after a transient provider failure', async () => {
+ const h = claimedServiceHarness();
+ const oldIntent: SkillUploadIntentRecord = {
+ skillId: h.skillId,
+ fileId: 'old-revision',
+ source: 'local',
+ storageKey: '/uploads/old-revision',
+ author: '507f1f77bcf86cd799439012',
+ state: 'committed',
+ };
+ h.persistence.listSkillDeletionTombstones.mockResolvedValue([]);
+ h.persistence.getSkillDeletion.mockResolvedValue(null);
+ h.persistence.listAbandonedSkillUploadIntents.mockResolvedValue([oldIntent]);
+ h.deleteFile.mockRejectedValueOnce(new Error('temporary storage outage'));
+ let releaseFirst!: () => void;
+ const firstAttempt = new Promise<void>((resolve) => {
+ releaseFirst = resolve;
+ });
+ h.persistence.releaseAbandonedSkillUploadIntentClaim.mockImplementationOnce(async () => {
+ releaseFirst();
+ return undefined;
+ });
+ const sweep = h.service.startSkillDeletionSweep();
+ await firstAttempt;
+ expect(h.persistence.markSkillUploadIntentCleaned).not.toHaveBeenCalled();
+ await new Promise((resolve) => setImmediate(resolve));
+ await sweep.runOnce();
+ sweep.stop();
+ expect(h.deleteFile).toHaveBeenCalledTimes(2);
+ expect(h.persistence.markSkillUploadIntentCleaned).toHaveBeenCalledWith('old-revision');
+});
+
it('marks deletion but cannot finalize until the writer-drain gate is enabled', async () => {
const h = claimedServiceHarness();
h.loadConfig.mockResolvedValue({
@@ -244,20 +284,22 @@ it('processes independent deletion tombstones with bounded concurrency', async (
);
h.persistence.claimSkillDeletion.mockResolvedValue(true);
h.persistence.refreshSkillDeletionClaim.mockResolvedValue(true);
- h.persistence.listSkillFiles.mockImplementation(async () => [
- {
- file_id: 'file-1',
- filepath: '/uploads/file-1',
- source: 'local',
- author: '507f1f77bcf86cd799439012',
- },
- {
- file_id: 'file-2',
- filepath: '/uploads/file-2',
- source: 'local',
- author: '507f1f77bcf86cd799439012',
- },
- ]);
+ h.persistence.iterateSkillCleanupFiles.mockImplementation(async function* () {
+ yield [
+ {
+ file_id: 'file-1',
+ filepath: '/uploads/file-1',
+ source: 'local',
+ author: '507f1f77bcf86cd799439012',
+ },
+ {
+ file_id: 'file-2',
+ filepath: '/uploads/file-2',
+ source: 'local',
+ author: '507f1f77bcf86cd799439012',
+ },
+ ];
+ });
h.persistence.deleteSkillFileAfterBlob.mockResolvedValue(true);
h.loadConfig.mockResolvedValue({
skillDeletion: { finalizationEnabled: true },
diff --git a/packages/api/src/skills/deletionService.ts b/packages/api/src/skills/deletionService.ts
index d6a00359b1..bca11932ac 100644
--- a/packages/api/src/skills/deletionService.ts
+++ b/packages/api/src/skills/deletionService.ts
@@ -202,6 +202,7 @@ export function createSkillDeletionService(deps: SkillDeletionServiceDeps): Skil
config?.fileConfig?.skills?.deletionCleanupConcurrency,
assertClaim,
journal,
+ config?.skillDeletion?.batchSize,
);
if (progress.cleanupComplete) {
await assertClaim();
diff --git a/packages/api/src/skills/handlers.spec.ts b/packages/api/src/skills/handlers.spec.ts
index 54ed7a2816..0c8624bace 100644
--- a/packages/api/src/skills/handlers.spec.ts
+++ b/packages/api/src/skills/handlers.spec.ts
@@ -2,8 +2,9 @@ import os from 'os';
import path from 'path';
import { Types } from 'mongoose';
import { promises as fs } from 'fs';
-import type { DeleteSkillResult, ISkillFile } from '@librechat/data-schemas';
+import type { ISkillFile } from '@librechat/data-schemas';
import type { Response } from 'express';
+import type { SkillDeletionProgress } from './deletion';
import type { SkillsHandlersDeps } from './handlers';
import type { ServerRequest } from '~/types';
import {
@@ -22,99 +23,70 @@ function mockResponse(): Response {
}
describe('skill delete handler', () => {
- it('retries dependent cleanup before the deleted skill becomes unreachable', async () => {
+ function deletionHarness() {
const id = new Types.ObjectId().toString();
- const deleteBlob = jest.fn(async () => undefined);
- const incomplete: DeleteSkillResult = {
+ const complete: SkillDeletionProgress = {
+ found: true,
deleted: true,
skillAbsent: true,
- cleanupComplete: false,
- failedCleanupSteps: ['skill_files'],
- };
- const complete: DeleteSkillResult = {
- deleted: false,
- skillAbsent: true,
cleanupComplete: true,
- failedCleanupSteps: [],
+ failedSteps: [],
};
- const deleteSkill = jest.fn().mockResolvedValueOnce(incomplete).mockResolvedValueOnce(complete);
- const file = {
- relativePath: 'references/query.sql',
- filepath: '/uploads/query.sql',
- source: 'local',
- author: new Types.ObjectId(),
- } as ISkillFile & { _id: Types.ObjectId };
- const handlers = createSkillsHandlers({
- deleteSkill,
- listSkillFiles: jest.fn(async () => [file]),
- getStrategyFunctions: jest.fn(() => ({ deleteFile: deleteBlob })),
+ const deps = {
+ deleteSkillById: jest.fn(async () => complete),
isValidObjectIdString: jest.fn(() => true),
- } as unknown as SkillsHandlersDeps);
- const req = { params: { id }, user: { id: 'manager-1' } } as unknown as ServerRequest;
+ };
+ return { id, deps };
+ }
+
+ it('delegates deletion through the claimed service before returning success', async () => {
+ const { id, deps } = deletionHarness();
+ const handlers = createSkillsHandlers(deps as unknown as SkillsHandlersDeps);
+ const req = { params: { id } } as unknown as ServerRequest;
const res = mockResponse();
await handlers.delete(req, res);
- expect(deleteSkill).toHaveBeenCalledTimes(2);
- expect(deleteBlob).toHaveBeenCalledTimes(1);
+ expect(deps.deleteSkillById).toHaveBeenCalledWith(id);
expect(res.status).toHaveBeenCalledWith(200);
expect(res.json).toHaveBeenCalledWith({ id, deleted: true, cleanupComplete: true });
});
- it('returns an evictable response when only non-file cleanup remains incomplete', async () => {
- const id = new Types.ObjectId().toString();
- const incomplete: DeleteSkillResult = {
- deleted: true,
- skillAbsent: true,
+ it('leaves the deletion pending when non-file cleanup is incomplete', async () => {
+ const { id, deps } = deletionHarness();
+ deps.deleteSkillById.mockResolvedValueOnce({
+ found: true,
+ deleted: false,
+ skillAbsent: false,
cleanupComplete: false,
- failedCleanupSteps: ['permissions'],
- };
- const deleteSkill = jest.fn(async () => incomplete);
- const handlers = createSkillsHandlers({
- deleteSkill,
- listSkillFiles: jest.fn(async () => []),
- getStrategyFunctions: jest.fn(),
- isValidObjectIdString: jest.fn(() => true),
- } as unknown as SkillsHandlersDeps);
+ failedSteps: ['permissions'],
+ });
+ const handlers = createSkillsHandlers(deps as unknown as SkillsHandlersDeps);
const req = { params: { id } } as unknown as ServerRequest;
const res = mockResponse();
await handlers.delete(req, res);
- expect(deleteSkill).toHaveBeenCalledTimes(2);
expect(res.status).toHaveBeenCalledWith(200);
expect(res.json).toHaveBeenCalledWith({ id, deleted: true, cleanupComplete: false });
});
- it('awaits blob cleanup and reports a partial deletion when storage rejects it', async () => {
- const id = new Types.ObjectId().toString();
- const deleteBlob = jest.fn(async () => {
- throw new Error('storage unavailable');
+ it('reports pending cleanup when another worker holds the claim', async () => {
+ const { id, deps } = deletionHarness();
+ deps.deleteSkillById.mockResolvedValueOnce({
+ found: true,
+ deleted: false,
+ skillAbsent: false,
+ cleanupComplete: false,
+ failedSteps: [],
});
- const handlers = createSkillsHandlers({
- deleteSkill: jest.fn(async () => ({
- deleted: true,
- skillAbsent: true,
- cleanupComplete: true,
- failedCleanupSteps: [],
- })),
- listSkillFiles: jest.fn(async () => [
- {
- relativePath: 'references/query.sql',
- filepath: '/uploads/query.sql',
- source: 'local',
- author: new Types.ObjectId(),
- } as ISkillFile & { _id: Types.ObjectId },
- ]),
- getStrategyFunctions: jest.fn(() => ({ deleteFile: deleteBlob })),
- isValidObjectIdString: jest.fn(() => true),
- } as unknown as SkillsHandlersDeps);
- const req = { params: { id }, user: { id: 'manager-1' } } as unknown as ServerRequest;
+ const handlers = createSkillsHandlers(deps as unknown as SkillsHandlersDeps);
+ const req = { params: { id } } as unknown as ServerRequest;
const res = mockResponse();
await handlers.delete(req, res);
- expect(deleteBlob).toHaveBeenCalledTimes(1);
+ expect(deps.deleteSkillById).toHaveBeenCalledWith(id);
expect(res.status).toHaveBeenCalledWith(200);
expect(res.json).toHaveBeenCalledWith({ id, deleted: true, cleanupComplete: false });
});
@@ -374,4 +346,22 @@ describe('skill file delete handler', () => {
expect(res.status).toHaveBeenCalledWith(409);
});
+
+ it('reports a conflict when whole-Skill deletion starts before the file row is removed', async () => {
+ const deleteBlob = jest.fn(async () => undefined);
+ const handlers = createSkillsHandlers({
+ getSkillFileByPath: jest.fn(async () => file),
+ deleteSkillFile: jest.fn(async () => {
+ throw Object.assign(new Error('Skill is being deleted'), { code: 'SKILL_DELETING' });
+ }),
+ getStrategyFunctions: jest.fn(() => ({ deleteFile: deleteBlob })),
+ } as unknown as SkillsHandlersDeps);
+ const res = mockResponse();
+
+ await handlers.deleteFile(req, res);
+
+ expect(deleteBlob).toHaveBeenCalledTimes(1);
+ expect(res.status).toHaveBeenCalledWith(409);
+ expect(res.json).toHaveBeenCalledWith({ error: 'SKILL_DELETING' });
+ });
});
diff --git a/packages/api/src/skills/handlers.ts b/packages/api/src/skills/handlers.ts
index d8e727b730..af667db212 100644
--- a/packages/api/src/skills/handlers.ts
+++ b/packages/api/src/skills/handlers.ts
@@ -34,12 +34,14 @@ import type {
import type { Response } from 'express';
import type { Types } from 'mongoose';
import type { ServerRequest, StrategyFunctions } from '~/types';
+import type { SkillDeletionProgress } from './deletion';
import { extractSkillContent, inspectContentWithTraversal } from '~/protection';
-import { deleteSkillWithRetry, mergeDeleteSkillResults } from './cleanup';
import { contentFilterBlockResponse } from '~/middleware/contentFilter';
+import { getSafeErrorMetadata } from '~/utils/errors';
import { getDeploymentSkillIds } from './deployment';
import { resolveDownloadPath } from '~/storage/path';
import { resolveSkillFilePathParam } from './path';
+import { deleteSkillWithRetry } from './cleanup';
import { parseSkillMarkdown } from './parse';
import { isBinaryBuffer } from './binary';
@@ -72,6 +74,7 @@ export interface SkillsHandlersDeps {
update: UpdateSkillInput;
}) => Promise<UpdateSkillResult>;
deleteSkill: (id: string) => Promise<DeleteSkillResult>;
+ deleteSkillById: (id: string) => Promise<SkillDeletionProgress>;
listSkillFiles: (
skillId: string | Types.ObjectId,
) => Promise<Array<ISkillFile & { _id: Types.ObjectId }>>;
@@ -337,6 +340,7 @@ export function createSkillsHandlers(deps: SkillsHandlersDeps): {
listSkillsByAccess,
updateSkill,
deleteSkill,
+ deleteSkillById,
listSkillFiles,
deleteSkillFile,
getSkillFileByPath,
@@ -609,66 +613,18 @@ export function createSkillsHandlers(deps: SkillsHandlersDeps): {
return res.status(400).json({ error: 'Invalid skill id' });
}
- // Collect file records before deletion so we can clean up storage blobs
- const files = await listSkillFiles(id);
-
- let result = await deleteSkill(id);
- if (result.skillAbsent && !result.cleanupComplete) {
- try {
- result = mergeDeleteSkillResults(result, await deleteSkill(id));
- } catch (error) {
- logger.error(`[deleteSkill] Cleanup retry failed for ${id}:`, error);
- }
- }
- /** Once SkillFile cleanup succeeds, the records loaded above are the
- * only remaining references to their blobs. Use them even if an
- * independent allowlist or permission cleanup step needs a retry. */
- let blobCleanupComplete = true;
- if (!result.failedCleanupSteps.includes('skill_files')) {
- const blobDeletions = files.map(async (file) => {
- const { deleteFile: deleteBlob } = getStrategyFunctions(file.source);
- if (!deleteBlob) {
- throw new Error(`No delete strategy for ${file.source}`);
- }
- await deleteBlob(skillFileDeletionRequest(req, file), {
- filepath: file.filepath,
- storageKey: file.storageKey,
- storageRegion: file.storageRegion,
- user: file.author?.toString?.(),
- tenantId: file.tenantId?.toString?.(),
- });
- });
- const blobResults = await Promise.allSettled(blobDeletions);
- for (const [index, blobResult] of blobResults.entries()) {
- if (blobResult.status === 'rejected') {
- blobCleanupComplete = false;
- logger.error(
- `[deleteSkill] Blob cleanup failed for ${files[index].relativePath}:`,
- blobResult.reason,
- );
- }
- }
- }
-
- const cleanupComplete = result.cleanupComplete && blobCleanupComplete;
- if (!cleanupComplete) {
- if (result.skillAbsent) {
- /** The client must evict the now-missing skill even though dependent
- * cleanup still needs repair. A 2xx response reaches the mutation's
- * cache reconciliation path; the flag preserves the warning. */
- const response: TDeleteSkillResponse = { id, deleted: true, cleanupComplete: false };
- return res.status(200).json(response);
- }
- return res.status(500).json({ error: 'Skill deletion cleanup did not finish' });
- }
- if (!result.deleted && files.length === 0) {
+ const progress = await deleteSkillById(id);
+ if (!progress.found) {
return res.status(404).json({ error: 'Skill not found' });
}
-
- const response: TDeleteSkillResponse = { id, deleted: true, cleanupComplete };
+ const response: TDeleteSkillResponse = {
+ id,
+ deleted: true,
+ cleanupComplete: progress.cleanupComplete,
+ };
return res.status(200).json(response);
} catch (error) {
- logger.error('[DELETE /skills/:id] Error deleting skill', error);
+ logger.error('[DELETE /skills/:id] Error deleting skill', getSafeErrorMetadata(error));
return res.status(500).json({ error: 'Error deleting skill' });
}
}
@@ -885,7 +841,10 @@ export function createSkillsHandlers(deps: SkillsHandlersDeps): {
};
return res.status(200).json(response);
} catch (error) {
- logger.error('[DELETE /skills/:id/files/*relativePath] Error', error);
+ if ((error as { code?: unknown })?.code === 'SKILL_DELETING') {
+ return res.status(409).json({ error: 'SKILL_DELETING' });
+ }
+ logger.error('[DELETE /skills/:id/files/*relativePath] Error', getSafeErrorMetadata(error));
return res.status(500).json({ error: 'Error deleting skill file' });
}
}
diff --git a/packages/api/src/skills/import.ts b/packages/api/src/skills/import.ts
index a4e660582f..0223871acb 100644
--- a/packages/api/src/skills/import.ts
+++ b/packages/api/src/skills/import.ts
@@ -27,6 +27,8 @@ import type {
TextContentFragment,
SkillContentInput,
} from '~/protection';
+import type { SkillBlobReservation } from './reservation';
+import type { SkillUploadIntentDeps } from './intent';
import type { ImportLimits } from './limits';
import {
inspectContent,
@@ -45,6 +47,7 @@ import { contentFilterBlockResponse } from '~/middleware/contentFilter';
import { resolveRequestTenantId } from '~/middleware/tenant';
import { createConcurrencyLimiter } from '~/utils/promise';
import { DEFAULT_SKILL_IMPORT_LIMITS } from './limits';
+import { withSkillUploadIntent } from './intent';
import { parseSkillMarkdown } from './parse';
import { isBinaryBuffer } from './binary';
@@ -130,22 +133,20 @@ function isDuplicateKeyError(error: unknown): boolean {
);
}
-export interface ImportSkillDeps {
+export interface ImportSkillDeps extends SkillUploadIntentDeps {
limits?: Partial<ImportLimits> | ((req: ServerRequest) => Partial<ImportLimits> | undefined);
createSkill: (data: CreateSkillInput) => Promise<CreateSkillResult>;
getSkillById: (id: string | Types.ObjectId) => Promise<(ISkill & { _id: Types.ObjectId }) | null>;
- deleteSkill: (id: string) => Promise<DeleteSkillResult>;
+ deleteSkill: (id: string) => Promise<DeleteSkillResult & { blobCleanupManaged?: boolean }>;
upsertSkillFile: (row: UpsertSkillFileInput) => Promise<ISkillFile & { _id: Types.ObjectId }>;
+ getSkillFileByPath: (skillId: string, relativePath: string) => Promise<ISkillFile | null>;
+ reserveBlob: (
+ req: Request,
+ params: { userId: string; fileName: string; isImage: boolean; tenantId?: string },
+ ) => Promise<SkillBlobReservation>;
saveBuffer: (
req: Request,
- params: {
- userId: string;
- buffer: Buffer;
- fileName: string;
- basePath?: string;
- isImage?: boolean;
- tenantId?: string;
- },
+ params: { reservation: SkillBlobReservation; buffer: Buffer },
) => Promise<{ filepath: string; source: string; storageKey?: string; storageRegion?: string }>;
deleteFile?: (
req: Request,
@@ -741,65 +742,68 @@ async function persistArchiveFile(
): Promise<void> {
const fileId = crypto.randomUUID();
const storageFileName = `${fileId}__${file.filename}`;
- const { filepath, source, storageKey, storageRegion } = await deps.saveBuffer(req, {
+ const reserved = await deps.reserveBlob(req, {
userId: context.userId,
- buffer,
fileName: storageFileName,
- basePath: 'uploads',
isImage: file.mimeType.startsWith('image/'),
tenantId: context.tenantId,
});
- const persistedBlob: PersistedArchiveBlob = {
+ if (!deps.deleteFile) throw new Error('Skill storage has no deletion strategy');
+ let saved: Awaited<ReturnType<ImportSkillDeps['saveBuffer']>> | undefined;
+ await withSkillUploadIntent(deps, {
+ skillId: context.skillId.toString(),
+ author: context.authorId.toString(),
+ tenantId: context.tenantId,
+ fileId,
+ source: reserved.source,
+ reservation: reserved,
+ save: async () => {
+ saved = await deps.saveBuffer(req, {
+ reservation: reserved,
+ buffer,
+ });
+ if (saved.source !== reserved.source) throw new Error('Skill storage changed during upload');
+ return saved.filepath;
+ },
+ persist: async (filepath) => {
+ await deps.upsertSkillFile({
+ skillId: context.skillId,
+ relativePath: file.relativePath,
+ file_id: fileId,
+ filename: file.filename,
+ filepath,
+ storageKey: saved?.storageKey,
+ storageRegion: saved?.storageRegion,
+ source: reserved.source,
+ mimeType: file.mimeType,
+ bytes: buffer.length,
+ isExecutable: false,
+ author: context.authorId,
+ tenantId: context.tenantId,
+ });
+ return fileId;
+ },
+ readPersisted: async () =>
+ (await deps.getSkillFileByPath(context.skillId.toString(), file.relativePath))?.file_id ===
+ fileId
+ ? fileId
+ : null,
+ deleteReserved: (ref, savedPath) =>
+ deps.deleteFile!(req, {
+ filepath: savedPath ?? ref.filepath,
+ storageKey: saved?.storageKey ?? ref.storageKey,
+ storageRegion: saved?.storageRegion ?? ref.storageRegion,
+ source: reserved.source,
+ user: context.authorId,
+ tenantId: context.tenantId,
+ }),
+ });
+ if (!saved) throw new Error('Skill upload returned no stored file');
+ context.persisted.push({
relativePath: file.relativePath,
- filepath,
- source,
- storageKey,
- storageRegion,
- rowPersisted: false,
- };
-
- try {
- await deps.upsertSkillFile({
- skillId: context.skillId,
- relativePath: file.relativePath,
- file_id: fileId,
- filename: file.filename,
- filepath,
- storageKey,
- storageRegion,
- source,
- mimeType: file.mimeType,
- bytes: buffer.length,
- isExecutable: false,
- author: context.authorId,
- tenantId: context.tenantId,
- });
- } catch (dbError) {
- let blobRemoved = false;
- if (deps.deleteFile) {
- await deps
- .deleteFile(req, {
- filepath,
- storageKey,
- storageRegion,
- source,
- user: context.authorId,
- tenantId: context.tenantId,
- })
- .then(() => {
- blobRemoved = true;
- })
- .catch((error) =>
- logger.error(`[importSkill] Orphan cleanup failed for ${file.relativePath}:`, error),
- );
- }
- if (!blobRemoved) {
- context.persisted.push(persistedBlob);
- }
- throw dbError;
- }
-
- context.persisted.push({ ...persistedBlob, rowPersisted: true });
+ ...saved,
+ rowPersisted: true,
+ });
}
async function cleanupArchiveBlobs(
@@ -867,7 +871,7 @@ async function rollbackArchiveImport(
context: ArchivePersistenceContext,
): Promise<{ skillRemoved: boolean; cleanupComplete: boolean }> {
const skillId = context.skillId.toString();
- let deletion: DeleteSkillResult;
+ let deletion: DeleteSkillResult & { blobCleanupManaged?: boolean };
try {
deletion = await deps.deleteSkill(skillId);
} catch (error) {
@@ -886,7 +890,11 @@ async function rollbackArchiveImport(
if (deletion.skillAbsent && !deletion.cleanupComplete) {
try {
- deletion = mergeDeleteSkillResults(deletion, await deps.deleteSkill(skillId));
+ const retried = await deps.deleteSkill(skillId);
+ deletion = {
+ ...mergeDeleteSkillResults(deletion, retried),
+ blobCleanupManaged: deletion.blobCleanupManaged || retried.blobCleanupManaged,
+ };
} catch (error) {
logger.error(`[importSkill] Rollback cleanup retry failed for skill ${skillId}:`, error);
}
@@ -914,9 +922,10 @@ async function rollbackArchiveImport(
}
const skillFileCleanupIncomplete = deletion.failedCleanupSteps.includes('skill_files');
- const blobsToDelete = skillFileCleanupIncomplete
- ? context.persisted.filter((blob) => !blob.rowPersisted)
- : context.persisted;
+ const blobsToDelete =
+ deletion.blobCleanupManaged || skillFileCleanupIncomplete
+ ? context.persisted.filter((blob) => !blob.rowPersisted)
+ : context.persisted;
const blobCleanupComplete = await cleanupArchiveBlobs(req, deps, context, blobsToDelete);
return {
diff --git a/packages/api/src/skills/index.ts b/packages/api/src/skills/index.ts
index 1d7ad05a4e..55ed0fce03 100644
--- a/packages/api/src/skills/index.ts
+++ b/packages/api/src/skills/index.ts
@@ -12,4 +12,7 @@ export * from './management';
export * from './upload';
export * from './save';
export * from './intent';
+export * from './deletion';
+export * from './deletionService';
+export * from './recoveryCli';
export * from './reservation';
diff --git a/packages/api/src/skills/management.spec.ts b/packages/api/src/skills/management.spec.ts
index 83e94568eb..fd0d360efd 100644
--- a/packages/api/src/skills/management.spec.ts
+++ b/packages/api/src/skills/management.spec.ts
@@ -3,14 +3,20 @@ import express from 'express';
import request from 'supertest';
import mongoose, { Types } from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
-import { createModels, createMethods, tenantStorage } from '@librechat/data-schemas';
import { PermissionBits, Permissions, PermissionTypes } from 'librechat-data-provider';
-import type { AllMethods, IRole, IUser } from '@librechat/data-schemas';
+import {
+ createModels,
+ createMethods,
+ createSkillDeletionPersistenceAdapter,
+ tenantStorage,
+} from '@librechat/data-schemas';
+import type { AllMethods, AppConfig, IRole, IUser } from '@librechat/data-schemas';
import type { FiltersConfig } from 'librechat-data-provider';
import type { RequestHandler } from 'express';
import type { SkillManagementDeps } from './management';
import type { SkillsHandlers } from './handlers';
import type { ServerRequest } from '~/types';
+import { createSkillDeletionService } from './deletionService';
import { createSkillManagementHandlers } from './management';
import { createSkillsHandlers } from './handlers';
@@ -87,8 +93,18 @@ beforeEach(async () => {
publiclyAccessible = jest.fn(async () => []);
publicStatus = jest.fn(async () => false);
capabilityCheck = jest.fn(async () => canManage);
+ const deletionService = createSkillDeletionService({
+ persistence: { ...db, ...createSkillDeletionPersistenceAdapter(db) },
+ loadConfig: async () =>
+ ({ skillDeletion: { finalizationEnabled: true } }) as unknown as AppConfig,
+ getStrategyFunctions: () => ({ deleteFile: async () => undefined }),
+ runAsSystem: async (callback) => callback(),
+ logger: { error: jest.fn(), warn: jest.fn() },
+ });
skillHandlers = createSkillsHandlers({
...db,
+ deleteSkillById: deletionService.deleteSkillById,
+ listSkillFiles: db.listSkillFiles,
findAccessibleResources: accessible,
findPubliclyAccessibleResources: publiclyAccessible,
hasPublicPermission: publicStatus,
@@ -484,6 +500,17 @@ it('returns a documented 409 when another writer changes the skill file during t
expect(result.body).toEqual({ error: { code: 'conflict', message: 'Resource conflict' } });
});
+it('returns a documented 409 when Skill deletion wins the file-write race', async () => {
+ saveFile.mockRejectedValueOnce(
+ Object.assign(new Error('Skill is being deleted'), { code: 'SKILL_DELETING' }),
+ );
+ const result = await request(app)
+ .put(`/skills/${skillId}/files/notes.md`)
+ .send({ content: 'Notes' })
+ .expect(409);
+ expect(result.body).toEqual({ error: { code: 'conflict', message: 'Resource conflict' } });
+});
+
it('returns safe errors for storage failures', async () => {
saveFile.mockRejectedValueOnce(new Error('secret-storage-path'));
const result = await request(app)
diff --git a/packages/api/src/skills/management.ts b/packages/api/src/skills/management.ts
index 41c9774ea8..3fd3aad455 100644
--- a/packages/api/src/skills/management.ts
+++ b/packages/api/src/skills/management.ts
@@ -433,7 +433,11 @@ export function createSkillManagementHandlers(
});
return res.status(200).json({ relativePath: result.relativePath, bytes: result.bytes });
} catch (error) {
- if (error instanceof Error && 'code' in error && error.code === 'SKILL_FILE_CONFLICT') {
+ if (
+ error instanceof Error &&
+ 'code' in error &&
+ (error.code === 'SKILL_FILE_CONFLICT' || error.code === 'SKILL_DELETING')
+ ) {
return sendError(res, 'conflict');
}
throw error;
diff --git a/packages/api/src/skills/recoveryCli.spec.ts b/packages/api/src/skills/recoveryCli.spec.ts
new file mode 100644
index 0000000000..48aebf8a32
--- /dev/null
+++ b/packages/api/src/skills/recoveryCli.spec.ts
@@ -0,0 +1,185 @@
+import { runSkillUploadRecoveryCli, runSkillUploadRecoveryCommand } from './recoveryCli';
+
+function harness() {
+ const messages: string[] = [];
+ const errors: string[] = [];
+ const deps = {
+ connect: jest.fn(async () => undefined),
+ disconnect: jest.fn(async () => undefined),
+ runAsSystem: async <T>(callback: () => Promise<T>) => callback(),
+ getIntent: jest.fn(
+ async (): Promise<{
+ fileId: string;
+ writerId: string;
+ skillId: string;
+ state: string;
+ cleanupHoldOwner?: string;
+ } | null> => ({
+ fileId: 'file-1',
+ writerId: 'writer-1',
+ skillId: 'skill-1',
+ state: 'committing',
+ }),
+ ),
+ listIntents: jest.fn(
+ async (): Promise<
+ Array<{
+ fileId: string;
+ writerId: string;
+ skillId: string;
+ state: string;
+ cleanupHoldOwner?: string;
+ }>
+ > => [
+ {
+ fileId: 'file-1',
+ writerId: 'writer-1',
+ skillId: 'skill-1',
+ state: 'committing',
+ },
+ ],
+ ),
+ recoverStoppedIntent: jest.fn(async () => ({
+ status: 'recovered' as const,
+ state: 'committed' as const,
+ })),
+ recoverStoppedCleanupHold: jest.fn(
+ async (): Promise<
+ { status: 'missing' | 'changed' } | { status: 'recovered'; cleanup: boolean }
+ > => ({
+ status: 'recovered' as const,
+ cleanup: true,
+ }),
+ ),
+ write: (message: string) => messages.push(message),
+ writeError: (message: string) => errors.push(message),
+ };
+ return { deps, messages, errors };
+}
+
+it('requires explicit writer-drain confirmation before changing an intent', async () => {
+ const h = harness();
+ expect(await runSkillUploadRecoveryCli(['--help'], h.deps)).toBe(0);
+ expect(await runSkillUploadRecoveryCli(['recover', 'file-1', 'writer-1'], h.deps)).toBe(2);
+ expect(h.deps.connect).not.toHaveBeenCalled();
+ expect(h.deps.recoverStoppedIntent).not.toHaveBeenCalled();
+ expect(h.errors[0]).toContain('Stop or drain all Skill upload and sync writers');
+});
+
+it('inspects and recovers a matching stopped writer through system context', async () => {
+ const h = harness();
+ expect(await runSkillUploadRecoveryCli(['inspect', 'file-1'], h.deps)).toBe(0);
+ expect(JSON.parse(h.messages[0])).toEqual({
+ fileId: 'file-1',
+ writerId: 'writer-1',
+ skillId: 'skill-1',
+ state: 'committing',
+ });
+ expect(await runSkillUploadRecoveryCli(['inspect-skill', 'skill-1'], h.deps)).toBe(0);
+ expect(JSON.parse(h.messages[1])).toEqual([
+ { fileId: 'file-1', writerId: 'writer-1', skillId: 'skill-1', state: 'committing' },
+ ]);
+ expect(
+ await runSkillUploadRecoveryCli(
+ ['recover', 'file-1', 'writer-1', '--confirm-writers-drained'],
+ h.deps,
+ ),
+ ).toBe(0);
+ expect(h.deps.recoverStoppedIntent).toHaveBeenCalledWith('file-1', 'writer-1');
+ expect(h.deps.disconnect).toHaveBeenCalledTimes(3);
+});
+
+it('shows the cleanup hold owner during inspection', async () => {
+ const h = harness();
+ h.deps.getIntent.mockResolvedValueOnce({
+ fileId: 'file-1',
+ writerId: 'original-writer',
+ skillId: 'skill-1',
+ state: 'committed',
+ cleanupHoldOwner: 'replacement-writer',
+ });
+ h.deps.listIntents.mockResolvedValueOnce([
+ {
+ fileId: 'file-1',
+ writerId: 'original-writer',
+ skillId: 'skill-1',
+ state: 'committed',
+ cleanupHoldOwner: 'replacement-writer',
+ },
+ ]);
+ expect(await runSkillUploadRecoveryCli(['inspect', 'file-1'], h.deps)).toBe(0);
+ expect(JSON.parse(h.messages[0])).toMatchObject({
+ writerId: 'original-writer',
+ cleanupHoldOwner: 'replacement-writer',
+ });
+ expect(await runSkillUploadRecoveryCli(['inspect-skill', 'skill-1'], h.deps)).toBe(0);
+ expect(JSON.parse(h.messages[1])[0]).toMatchObject({
+ writerId: 'original-writer',
+ cleanupHoldOwner: 'replacement-writer',
+ });
+});
+
+it('requires writer-drain confirmation before recovering a cleanup hold', async () => {
+ const h = harness();
+ expect(await runSkillUploadRecoveryCli(['recover-cleanup', 'file-1', 'hold-1'], h.deps)).toBe(2);
+ expect(
+ await runSkillUploadRecoveryCli(
+ ['recover-cleanup', 'file-1', 'hold-1', '--confirm-writers-active'],
+ h.deps,
+ ),
+ ).toBe(2);
+ expect(h.deps.connect).not.toHaveBeenCalled();
+ expect(h.deps.recoverStoppedCleanupHold).not.toHaveBeenCalled();
+});
+
+it.each([
+ [true, 'cleanup queued'],
+ [false, 'file retained'],
+] as const)('reports recovered cleanup hold outcome %s', async (cleanup, outcome) => {
+ const h = harness();
+ h.deps.recoverStoppedCleanupHold.mockResolvedValueOnce({ status: 'recovered', cleanup });
+ expect(
+ await runSkillUploadRecoveryCli(
+ ['recover-cleanup', 'file-1', 'hold-1', '--confirm-writers-drained'],
+ h.deps,
+ ),
+ ).toBe(0);
+ expect(h.deps.recoverStoppedCleanupHold).toHaveBeenCalledWith('file-1', 'hold-1');
+ expect(h.messages).toEqual([`Recovered cleanup hold file-1; ${outcome}`]);
+ expect(h.deps.disconnect).toHaveBeenCalledTimes(1);
+});
+
+it.each(['missing', 'changed'] as const)(
+ 'does not claim recovery for a %s cleanup hold',
+ async (status) => {
+ const h = harness();
+ h.deps.recoverStoppedCleanupHold.mockResolvedValueOnce({ status });
+ expect(
+ await runSkillUploadRecoveryCli(
+ ['recover-cleanup', 'file-1', 'hold-1', '--confirm-writers-drained'],
+ h.deps,
+ ),
+ ).toBe(1);
+ expect(h.errors).toEqual([
+ `Upload cleanup hold file-1 was ${status}; inspect it again before retrying`,
+ ]);
+ expect(h.deps.disconnect).toHaveBeenCalledTimes(1);
+ },
+);
+
+it.each(['connect', 'getIntent', 'disconnect'] as const)(
+ 'reports bounded metadata when %s fails without logging driver details',
+ async (step) => {
+ const h = harness();
+ h.deps[step].mockRejectedValueOnce(
+ Object.assign(new Error('mongodb://user:[email protected]/private'), {
+ status: 503,
+ query: { token: 'secret' },
+ }),
+ );
+
+ expect(await runSkillUploadRecoveryCommand(['inspect', 'file-1'], h.deps)).toBe(1);
+ expect(h.errors).toEqual(['{"type":"Error","status":503}']);
+ expect(JSON.stringify(h.errors)).not.toContain('secret');
+ },
+);
diff --git a/packages/api/src/skills/recoveryCli.ts b/packages/api/src/skills/recoveryCli.ts
new file mode 100644
index 0000000000..f3c625a7e8
--- /dev/null
+++ b/packages/api/src/skills/recoveryCli.ts
@@ -0,0 +1,145 @@
+import { getSafeErrorMetadata } from '~/utils/errors';
+
+interface RecoveryIntent {
+ fileId: string;
+ writerId: string;
+ state: string;
+ skillId: { toString(): string };
+ cleanupHoldOwner?: string;
+}
+
+interface RecoveryResult {
+ status: 'missing' | 'changed' | 'recovered';
+ state?: 'committed' | 'abandoned';
+}
+
+type CleanupHoldRecoveryResult =
+ | { status: 'missing' | 'changed' }
+ | { status: 'recovered'; cleanup: boolean };
+
+export interface SkillUploadRecoveryCliDeps {
+ connect: () => Promise<unknown>;
+ disconnect: () => Promise<unknown>;
+ runAsSystem: <T>(callback: () => Promise<T>) => Promise<T>;
+ getIntent: (fileId: string) => Promise<RecoveryIntent | null>;
+ listIntents: (skillId: string) => Promise<RecoveryIntent[]>;
+ recoverStoppedIntent: (fileId: string, writerId: string) => Promise<RecoveryResult>;
+ recoverStoppedCleanupHold: (
+ fileId: string,
+ holdOwner: string,
+ ) => Promise<CleanupHoldRecoveryResult>;
+ write: (message: string) => void;
+ writeError: (message: string) => void;
+}
+
+const usage =
+ 'Usage: recover-skill-upload inspect-skill <skillId> | inspect <fileId> | recover <fileId> <writerId> --confirm-writers-drained | recover-cleanup <fileId> <holdOwner> --confirm-writers-drained. Stop or drain all Skill upload and sync writers before recovery.';
+
+/** Operator entry point: the confirmation means all Skill upload and sync writers were drained. */
+export async function runSkillUploadRecoveryCli(
+ args: string[],
+ deps: SkillUploadRecoveryCliDeps,
+): Promise<number> {
+ const [action, fileId, owner, confirmation] = args;
+ const inspect = action === 'inspect' && args.length === 2 && Boolean(fileId);
+ const inspectSkill = action === 'inspect-skill' && args.length === 2 && Boolean(fileId);
+ const recover =
+ action === 'recover' &&
+ args.length === 4 &&
+ Boolean(fileId) &&
+ Boolean(owner) &&
+ confirmation === '--confirm-writers-drained';
+ const recoverCleanup =
+ action === 'recover-cleanup' &&
+ args.length === 4 &&
+ Boolean(fileId) &&
+ Boolean(owner) &&
+ confirmation === '--confirm-writers-drained';
+ if (args.length === 1 && (action === '--help' || action === '-h')) {
+ deps.write(usage);
+ return 0;
+ }
+ if (!inspect && !inspectSkill && !recover && !recoverCleanup) {
+ deps.writeError(usage);
+ return 2;
+ }
+
+ let connected = false;
+ try {
+ await deps.connect();
+ connected = true;
+ if (inspectSkill) {
+ const intents = await deps.runAsSystem(() => deps.listIntents(fileId));
+ deps.write(
+ JSON.stringify(
+ intents.map((intent) => ({
+ fileId: intent.fileId,
+ writerId: intent.writerId,
+ skillId: intent.skillId.toString(),
+ state: intent.state,
+ cleanupHoldOwner: intent.cleanupHoldOwner,
+ })),
+ ),
+ );
+ return 0;
+ }
+ if (inspect) {
+ const intent = await deps.runAsSystem(() => deps.getIntent(fileId));
+ if (!intent) {
+ deps.writeError(`No upload intent found for ${fileId}`);
+ return 1;
+ }
+ deps.write(
+ JSON.stringify({
+ fileId: intent.fileId,
+ writerId: intent.writerId,
+ skillId: intent.skillId.toString(),
+ state: intent.state,
+ cleanupHoldOwner: intent.cleanupHoldOwner,
+ }),
+ );
+ return 0;
+ }
+
+ if (recoverCleanup) {
+ const result = await deps.runAsSystem(() => deps.recoverStoppedCleanupHold(fileId, owner));
+ if (result.status !== 'recovered') {
+ deps.writeError(
+ `Upload cleanup hold ${fileId} was ${result.status}; inspect it again before retrying`,
+ );
+ return 1;
+ }
+ deps.write(
+ result.cleanup
+ ? `Recovered cleanup hold ${fileId}; cleanup queued`
+ : `Recovered cleanup hold ${fileId}; file retained`,
+ );
+ return 0;
+ }
+
+ const result = await deps.runAsSystem(() => deps.recoverStoppedIntent(fileId, owner));
+ if (result.status !== 'recovered') {
+ deps.writeError(
+ `Upload intent ${fileId} was ${result.status}; inspect it again before retrying`,
+ );
+ return 1;
+ }
+ deps.write(`Recovered ${fileId} as ${result.state}`);
+ return 0;
+ } finally {
+ if (connected) await deps.disconnect();
+ }
+}
+
+/** Keep operational errors at the process boundary without exposing driver details. */
+export async function runSkillUploadRecoveryCommand(
+ args: string[],
+ deps: SkillUploadRecoveryCliDeps,
+): Promise<number> {
+ try {
+ return await runSkillUploadRecoveryCli(args, deps);
+ } catch (error) {
+ deps.writeError(JSON.stringify(getSafeErrorMetadata(error)));
+ return 1;
+ }
+}
diff --git a/packages/api/src/skills/save.spec.ts b/packages/api/src/skills/save.spec.ts
index 49200d1cda..89be733d7b 100644
--- a/packages/api/src/skills/save.spec.ts
+++ b/packages/api/src/skills/save.spec.ts
@@ -1,3 +1,4 @@
+import { Types } from 'mongoose';
import { FileSources } from 'librechat-data-provider';
import type { SaveBufferParams } from '~/storage/types';
import { createSkillFileSaver, createSkillManagementFileSaver } from './save';
@@ -16,7 +17,9 @@ const existing = {
};
function harness() {
- const saveBuffer = jest.fn(async (_input: SaveBufferParams) => '/uploads/new');
+ const saveBuffer = jest.fn(
+ async (input: SaveBufferParams) => `/uploads/${input.userId}/${input.fileName}`,
+ );
const deleteFile = jest.fn(async () => undefined);
const getSkillFileByPath = jest.fn(
async (): ReturnType<SaverDeps['getSkillFileByPath']> => existing,
@@ -27,7 +30,7 @@ function harness() {
): ReturnType<SaverDeps['upsertSkillFile']> => ({
...existing,
file_id: row.file_id,
- filepath: '/uploads/new',
+ filepath: row.filepath,
bytes: 9,
}),
);
@@ -36,6 +39,13 @@ function harness() {
upsertSkillFile,
resolveStorage: jest.fn(() => ({ source: FileSources.local, saveBuffer })),
getStrategyFunctions: jest.fn(() => ({ deleteFile })),
+ reserveSkillUploadIntent: jest.fn(async (input) => ({
+ ...input,
+ skillId: new Types.ObjectId(),
+ author: new Types.ObjectId(),
+ state: 'uploading' as const,
+ })),
+ transitionSkillUploadIntent: jest.fn(async () => true),
} satisfies SaverDeps;
const req = { user: { id: 'editor', _id: 'editor' }, config: {} } as never;
const params = {
@@ -62,6 +72,11 @@ function harness() {
}
describe('management skill file saver', () => {
+ it('leaves superseded blob cleanup to the durable persistence queue', async () => {
+ const h = harness();
+ await h.saveManagement(h.managementParams);
+ expect(h.deleteFile).not.toHaveBeenCalled();
+ });
it('creates a missing path with an insert-only write and no second lookup', async () => {
const h = harness();
h.getSkillFileByPath.mockResolvedValueOnce(null);
@@ -86,10 +101,7 @@ describe('management skill file saver', () => {
expect(h.upsertSkillFile).toHaveBeenCalledWith(
expect.objectContaining({ createOnly: false, expectedFileId: 'initial' }),
);
- expect(h.deleteFile).toHaveBeenCalledWith(
- h.params.req,
- expect.objectContaining({ filepath: '/uploads/initial', user: 'original-owner' }),
- );
+ expect(h.deleteFile).not.toHaveBeenCalled();
});
it('rejects and removes only its upload if a concurrent browser edit wins', async () => {
@@ -109,7 +121,10 @@ describe('management skill file saver', () => {
expect(h.deleteFile).toHaveBeenCalledTimes(1);
expect(h.deleteFile).toHaveBeenCalledWith(
h.params.req,
- expect.objectContaining({ filepath: '/uploads/new', user: 'editor' }),
+ expect.objectContaining({
+ filepath: expect.stringMatching(/^\/uploads\/editor\//),
+ user: 'editor',
+ }),
);
});
});
@@ -146,7 +161,7 @@ describe('agent skill file saver', () => {
);
});
- it('preserves executable metadata and deletes the original blob under its owner on replacement', async () => {
+ it('preserves executable metadata and queues the original blob through persistence', async () => {
const h = harness();
await expect(h.save(h.params)).resolves.toEqual({
bytes: 9,
@@ -155,13 +170,7 @@ describe('agent skill file saver', () => {
expect(h.upsertSkillFile).toHaveBeenCalledWith(
expect.objectContaining({ expectedFileId: 'initial', createOnly: false, isExecutable: true }),
);
- expect(h.deleteFile).toHaveBeenCalledWith(h.params.req, {
- filepath: '/uploads/initial',
- storageKey: undefined,
- storageRegion: undefined,
- user: 'original-owner',
- tenantId: 'original-tenant',
- });
+ expect(h.deleteFile).not.toHaveBeenCalled();
expect(h.deleteFile).not.toHaveBeenCalledWith(
h.params.req,
expect.objectContaining({ filepath: '/uploads/new' }),
@@ -181,7 +190,10 @@ describe('agent skill file saver', () => {
await expect(h.save(h.params)).rejects.toMatchObject({ code: 'SKILL_FILE_CONFLICT' });
expect(h.deleteFile).toHaveBeenCalledWith(
h.params.req,
- expect.objectContaining({ filepath: '/uploads/new', user: 'editor' }),
+ expect.objectContaining({
+ filepath: expect.stringMatching(/^\/uploads\/editor\//),
+ user: 'editor',
+ }),
);
expect(h.deleteFile).not.toHaveBeenCalledWith(
h.params.req,
@@ -189,24 +201,24 @@ describe('agent skill file saver', () => {
);
});
- it('retains committed bytes and cleans up the superseded blob if the parent bump fails', async () => {
+ it('retains committed bytes if the parent bump fails', async () => {
const h = harness();
let committedRevision = '';
h.upsertSkillFile.mockImplementationOnce(async (row) => {
committedRevision = row.file_id;
throw new Error('parent update failed');
});
- h.getSkillFileByPath.mockResolvedValueOnce(existing).mockImplementationOnce(async () => ({
+ h.getSkillFileByPath.mockResolvedValueOnce(existing).mockImplementation(async () => ({
...existing,
file_id: committedRevision,
- filepath: '/uploads/new',
+ filepath: '/uploads/editor/committed',
}));
- await expect(h.save(h.params)).rejects.toThrow('parent update failed');
+ await expect(h.save(h.params)).resolves.toEqual({
+ bytes: h.params.content.length,
+ relativePath: h.params.relativePath,
+ });
expect(committedRevision).not.toBe('');
- expect(h.deleteFile).toHaveBeenCalledWith(
- h.params.req,
- expect.objectContaining({ filepath: '/uploads/initial', user: 'original-owner' }),
- );
+ expect(h.deleteFile).not.toHaveBeenCalled();
expect(h.deleteFile).not.toHaveBeenCalledWith(
h.params.req,
expect.objectContaining({ filepath: '/uploads/new' }),
@@ -224,7 +236,10 @@ describe('agent skill file saver', () => {
});
expect(h.deleteFile).toHaveBeenCalledWith(
h.params.req,
- expect.objectContaining({ filepath: '/uploads/new', user: 'editor' }),
+ expect.objectContaining({
+ filepath: expect.stringMatching(/^\/uploads\/editor\//),
+ user: 'editor',
+ }),
);
});
});
diff --git a/packages/api/src/skills/save.ts b/packages/api/src/skills/save.ts
index 66b434e7e1..ea2a43ce2f 100644
--- a/packages/api/src/skills/save.ts
+++ b/packages/api/src/skills/save.ts
@@ -1,14 +1,16 @@
import { randomUUID } from 'node:crypto';
-import { logger } from '@librechat/data-schemas';
import type { UpsertSkillFileInput } from '@librechat/data-schemas';
import type { TSkillFile } from 'librechat-data-provider';
import type { ServerRequest, StrategyFunctions } from '~/types';
import type { SaveBufferParams } from '~/storage/types';
+import type { SkillUploadIntentDeps } from './intent';
import type { StoredSkillFile } from './upload';
import { resolveRequestTenantId } from '~/middleware/tenant';
import { getStorageMetadata } from '~/storage/metadata';
+import { withSkillUploadIntent } from './intent';
+import { reserveSkillBlob } from './reservation';
-interface SkillFileSaveDeps {
+interface SkillFileSaveDeps extends SkillUploadIntentDeps {
getSkillFileByPath: (skillId: string, relativePath: string) => Promise<StoredSkillFile | null>;
upsertSkillFile: (
row: Omit<UpsertSkillFileInput, 'skillId' | 'author'> & { skillId: string; author: string },
@@ -83,75 +85,68 @@ async function saveSkillFile(
const filename = relativePath.slice(relativePath.lastIndexOf('/') + 1);
const buffer = Buffer.from(content, 'utf8');
const storage = deps.resolveStorage(req, { isImage: mimeType.startsWith('image/') });
- const filepath = await storage.saveBuffer({
+ const strategy = deps.getStrategyFunctions(storage.source) as Partial<StrategyFunctions> & {
+ getFileURL?: (params: {
+ userId: string;
+ fileName: string;
+ basePath: string;
+ tenantId?: string;
+ }) => Promise<string>;
+ };
+ if (!strategy.deleteFile) throw new Error('Skill storage has no deletion strategy');
+ const fileName = `${fileId}__${filename}`;
+ const reservation = await reserveSkillBlob({
+ source: storage.source,
userId,
- buffer,
- fileName: `${fileId}__${filename}`,
- basePath: 'uploads',
+ fileName,
tenantId,
+ getFileURL: strategy.getFileURL,
});
- const storageMetadata = getStorageMetadata({ filepath, source: storage.source });
- const cleanupPreviousBlob = (): void => {
- if (!existingFile || existingFile.filepath === filepath) {
- return;
- }
- const deleteFile = deps.getStrategyFunctions(existingFile.source).deleteFile;
- if (deleteFile) {
- deleteFile(req, {
- filepath: existingFile.filepath,
- storageKey: existingFile.storageKey,
- storageRegion: existingFile.storageRegion,
- user: existingFile.author?.toString() ?? userId,
- tenantId: existingFile.tenantId ?? tenantId,
- }).catch((error: Error) =>
- logger.error('[saveSkillFileContent] Old blob cleanup failed:', error),
- );
- }
- };
-
- let result: StoredSkillFile & { bytes: number; relativePath: string };
- try {
- const saved = await deps.upsertSkillFile({
- skillId,
- relativePath,
- expectedFileId: revision,
- createOnly: insertOnly,
- file_id: fileId,
- filename,
- filepath,
- ...storageMetadata,
- source: storage.source,
- mimeType,
- bytes: buffer.length,
- isExecutable: existingFile?.isExecutable ?? false,
- author: user._id?.toString() ?? userId,
- tenantId,
- });
- if (!saved) {
- throw Object.assign(new Error('Skill file save failed to persist metadata'), {
- code: 'SKILL_FILE_UPSERT_NOT_FOUND',
+ const result = await withSkillUploadIntent(deps, {
+ skillId,
+ author: user._id?.toString() ?? userId,
+ tenantId,
+ fileId,
+ source: storage.source,
+ reservation,
+ save: () => storage.saveBuffer({ ...reservation.writeLocation, buffer }),
+ persist: async (filepath) => {
+ const row = await deps.upsertSkillFile({
+ skillId,
+ relativePath,
+ expectedFileId: revision,
+ createOnly: insertOnly,
+ file_id: fileId,
+ filename,
+ filepath,
+ ...getStorageMetadata({ filepath, source: storage.source }),
+ source: storage.source,
+ mimeType,
+ bytes: buffer.length,
+ isExecutable: existingFile?.isExecutable ?? false,
+ author: user._id?.toString() ?? userId,
+ tenantId,
});
- }
- result = saved;
- } catch (error) {
- try {
- const persisted = await deps.getSkillFileByPath(skillId, relativePath);
- if (persisted?.file_id === fileId) {
- cleanupPreviousBlob();
- } else {
- await deps.getStrategyFunctions(storage.source).deleteFile?.(req, {
- filepath,
- ...storageMetadata,
- user: userId,
- tenantId,
+ if (!row) {
+ throw Object.assign(new Error('Skill file upsert returned no row'), {
+ code: 'SKILL_FILE_UPSERT_NOT_FOUND',
});
}
- } catch (cleanupError) {
- logger.error('[saveSkillFileContent] Failed to clean up uploaded blob:', cleanupError);
- }
- throw error;
- }
-
- cleanupPreviousBlob();
+ return { filepath: row.filepath, bytes: buffer.length, relativePath };
+ },
+ readPersisted: async () => {
+ const persisted = await deps.getSkillFileByPath(skillId, relativePath);
+ return persisted?.file_id === fileId
+ ? { filepath: persisted.filepath, bytes: buffer.length, relativePath }
+ : null;
+ },
+ deleteReserved: (file, savedPath) =>
+ strategy.deleteFile!(req, {
+ filepath: savedPath ?? file.filepath,
+ ...getStorageMetadata({ filepath: savedPath ?? file.filepath, source: storage.source }),
+ user: userId,
+ tenantId,
+ }),
+ });
return { bytes: result.bytes, relativePath: result.relativePath };
}
diff --git a/packages/api/src/skills/sync/github.spec.ts b/packages/api/src/skills/sync/github.spec.ts
index 1063d3bb4b..3781c2349f 100644
--- a/packages/api/src/skills/sync/github.spec.ts
+++ b/packages/api/src/skills/sync/github.spec.ts
@@ -1,7 +1,13 @@
import crypto from 'crypto';
import mongoose, { Types } from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
-import { logger, getTenantId, createModels, createMethods } from '@librechat/data-schemas';
+import {
+ logger,
+ getTenantId,
+ createModels,
+ createMethods,
+ createSkillDeletionPersistenceAdapter,
+} from '@librechat/data-schemas';
import type {
ISkill,
ISkillFile,
@@ -13,11 +19,14 @@ import type {
UpdateSkillResult,
UpsertSkillFileInput,
DeleteSkillResult,
+ AppConfig,
} from '@librechat/data-schemas';
import type { RepoTreeEntry, GitRepoAdapter } from './adapters/types';
import type { GitHubSkillSyncDeps } from './github';
+import { createSkillDeletionService } from '../deletionService';
import { DEFAULT_SKILL_IMPORT_LIMITS } from '../limits';
import { createGitHubSkillSyncRunner } from './github';
+import { runSkillDeletion } from '../deletion';
function response(body: unknown, status = 200, headers: Record<string, string> = {}): Response {
const normalizedHeaders = new Map(
@@ -329,6 +338,21 @@ function createDeps(
}),
deleteSkillFile: jest.fn(async () => ({ deleted: true })),
deleteSkill: jest.fn(async () => completeSkillDeletion(true)),
+ reserveBlob: jest.fn(async ({ userId, fileName, tenantId }) => ({
+ filepath: '/uploads/file-id__run.sh',
+ source: 'local',
+ writeLocation: { userId, fileName, basePath: 'uploads' as const, tenantId },
+ })),
+ reserveSkillUploadIntent: jest.fn(async (input) => ({
+ ...input,
+ skillId: new Types.ObjectId(input.skillId),
+ author: new Types.ObjectId(input.author),
+ state: 'uploading' as const,
+ })),
+ transitionSkillUploadIntent: jest.fn(async () => true),
+ queueCommittedSkillUploadCleanup: jest.fn(async () => undefined),
+ finishSkillUploadCleanupHold: jest.fn(async () => true),
+ markSkillUploadIntentCleaned: jest.fn(async () => undefined),
saveBuffer: jest.fn(async () => ({ filepath: '/uploads/file-id__run.sh', source: 'local' })),
deleteFile: jest.fn(async () => undefined),
grantPermission: jest.fn(async () => undefined),
@@ -1815,7 +1839,7 @@ describe('createGitHubSkillSyncRunner', () => {
);
});
- it('fails the source when recreating a stale mirror cannot preserve its dependent state', async () => {
+ it('fails the source without recreating a stale mirror whose blobs were deleted', async () => {
const staleId = new Types.ObjectId();
const existingId = new Types.ObjectId();
const author = makeSourceAuthorId();
@@ -1869,18 +1893,9 @@ describe('createGitHubSkillSyncRunner', () => {
expect(result.status).toBe('failed');
expect(deleteSkill).toHaveBeenCalledWith(staleId.toString());
expect(deleteSkill).toHaveBeenCalledTimes(1);
- expect(createSkill).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'renamed',
- sourceMetadata: expect.objectContaining({
- upstreamId: 'librechat-skills:skills/removed',
- }),
- }),
- );
- expect(deps.grantPermission).toHaveBeenCalledWith(
- expect.objectContaining({ resourceId: restoredSkill?._id }),
- );
- expect(persistedSkills.has(restoredSkill?._id.toString() ?? '')).toBe(true);
+ expect(createSkill).not.toHaveBeenCalled();
+ expect(restoredSkill).toBeUndefined();
+ expect(persistedSkills.has(staleId.toString())).toBe(false);
expect(deps.upsertStatus).toHaveBeenLastCalledWith(
expect.objectContaining({
status: 'failed',
@@ -1894,7 +1909,7 @@ describe('createGitHubSkillSyncRunner', () => {
it.each([
{ failedStep: 'skill_files' as const, expectedBlobDeletes: 0 },
- { failedStep: 'permissions' as const, expectedBlobDeletes: 1 },
+ { failedStep: 'permissions' as const, expectedBlobDeletes: 0 },
])(
'fails the source and safely handles blobs when stale deletion leaves $failedStep incomplete',
async ({ failedStep, expectedBlobDeletes }) => {
@@ -2548,12 +2563,12 @@ describe('createGitHubSkillSyncRunner', () => {
);
});
- it('fails the source when rollback cannot remove a stored skill file', async () => {
+ it('fails the source when durable rollback cannot finish storage cleanup', async () => {
const storedFiles: Array<ISkillFile & { _id: Types.ObjectId }> = [];
const deleteFile = jest.fn(async () => {
throw new Error('storage cleanup unavailable');
});
- const deleteSkill = jest.fn(async () => completeSkillDeletion(true));
+ const deleteSkill = jest.fn(async () => incompleteSkillDeletion(['skill_files']));
const deps = createDeps({
listSkillFiles: jest.fn(async () => storedFiles),
upsertSkillFile: jest.fn(async (input: UpsertSkillFileInput) => {
@@ -2573,7 +2588,7 @@ describe('createGitHubSkillSyncRunner', () => {
const result = await runner.runOnce();
expect(result.status).toBe('failed');
- expect(deleteFile).toHaveBeenCalledTimes(1);
+ expect(deleteFile).not.toHaveBeenCalled();
expect(deleteSkill).toHaveBeenCalledTimes(1);
expect(deps.upsertStatus).toHaveBeenLastCalledWith(
expect.objectContaining({
@@ -2584,7 +2599,7 @@ describe('createGitHubSkillSyncRunner', () => {
);
});
- it('fails the source when an unpersisted upload cannot be cleaned up', async () => {
+ it('retains an unpersisted upload intent when immediate cleanup fails', async () => {
const deleteFile = jest.fn(async () => {
throw new Error('orphan cleanup unavailable');
});
@@ -2603,8 +2618,8 @@ describe('createGitHubSkillSyncRunner', () => {
expect(deps.upsertStatus).toHaveBeenLastCalledWith(
expect.objectContaining({
status: 'failed',
- errorCode: 'SYNC_ROLLBACK_FAILED',
- errorMessage: 'Rollback failed after: database unavailable',
+ errorCode: 'SYNC_FAILED',
+ errorMessage: 'database unavailable',
}),
);
});
@@ -2971,93 +2986,115 @@ describe('createGitHubSkillSyncRunner', () => {
expect(deps.updateSkill).not.toHaveBeenCalled();
});
- it('restores existing skill files when the skill update fails after file sync', async () => {
- const existing = makeSkill({
- name: 'research',
- description: 'Old description',
- body: 'Old body',
- author: new Types.ObjectId(),
- authorName: 'GitHub Sync',
- source: 'github',
- sourceMetadata: {
- provider: 'github',
- sourceId: 'librechat-skills',
- upstreamId: 'librechat-skills:skills/research',
- owner: 'LibreChat',
- repo: 'skills',
- ref: 'main',
- skillPath: 'skills/research',
- },
- }) as ISkill & { _id: Types.ObjectId };
- const oldFile = makeSkillFile(existing);
- const files = new Map<string, ISkillFile & { _id: Types.ObjectId }>([
- [oldFile.relativePath, oldFile],
- ]);
- const upsertSkillFile = jest.fn(
- async (
- row: Parameters<GitHubSkillSyncDeps['upsertSkillFile']>[0],
- ): Promise<ISkillFile & { _id: Types.ObjectId }> => {
- const current = files.get(row.relativePath);
- const next = {
- _id: current?._id ?? new Types.ObjectId(),
- skillId: row.skillId as Types.ObjectId,
- relativePath: row.relativePath,
- file_id: row.file_id,
- filename: row.filename,
- filepath: row.filepath,
- storageKey: row.storageKey,
- storageRegion: row.storageRegion,
- source: row.source,
- sourceMetadata: row.sourceMetadata,
- mimeType: row.mimeType,
- bytes: row.bytes,
- category: 'script' as const,
- isExecutable: row.isExecutable ?? false,
- author: new Types.ObjectId(row.author),
- tenantId: row.tenantId,
- };
- files.set(row.relativePath, next);
- return next;
- },
- );
- const deps = createDeps({
- findSkillBySourceIdentity: jest.fn(async () => existing),
- getSkillById: jest.fn(async () => ({ ...existing, version: existing.version + 1 })),
- getSkillFileByPath: jest.fn(
- async (_skillId, relativePath) => files.get(relativePath) ?? null,
- ),
- listSkillFiles: jest.fn(async () => Array.from(files.values())),
- upsertSkillFile,
- deleteSkillFile: jest.fn(async (_skillId, relativePath) => ({
- deleted: files.delete(relativePath),
- })),
- saveBuffer: jest.fn(async () => ({
- filepath: '/uploads/new-file-id__run.sh',
- source: 'local',
- })),
- deleteFile: jest.fn(async () => undefined),
- updateSkill: jest.fn(async () => ({ status: 'conflict' as const, current: existing })),
- });
- const runner = createGitHubSkillSyncRunner(deps);
- const result = await runner.runOnce();
+ it.each([false, true])(
+ 'retains a recoverable file when sync rollback cleanup queue fails: queueFailed=%s',
+ async (queueFailed) => {
+ const existing = makeSkill({
+ name: 'research',
+ description: 'Old description',
+ body: 'Old body',
+ author: new Types.ObjectId(),
+ authorName: 'GitHub Sync',
+ source: 'github',
+ sourceMetadata: {
+ provider: 'github',
+ sourceId: 'librechat-skills',
+ upstreamId: 'librechat-skills:skills/research',
+ owner: 'LibreChat',
+ repo: 'skills',
+ ref: 'main',
+ skillPath: 'skills/research',
+ },
+ }) as ISkill & { _id: Types.ObjectId };
+ const oldFile = makeSkillFile(existing);
+ const files = new Map<string, ISkillFile & { _id: Types.ObjectId }>([
+ [oldFile.relativePath, oldFile],
+ ]);
+ const upsertSkillFile = jest.fn(
+ async (
+ row: Parameters<GitHubSkillSyncDeps['upsertSkillFile']>[0],
+ ): Promise<ISkillFile & { _id: Types.ObjectId }> => {
+ const current = files.get(row.relativePath);
+ const next = {
+ _id: current?._id ?? new Types.ObjectId(),
+ skillId: row.skillId as Types.ObjectId,
+ relativePath: row.relativePath,
+ file_id: row.file_id,
+ filename: row.filename,
+ filepath: row.filepath,
+ storageKey: row.storageKey,
+ storageRegion: row.storageRegion,
+ source: row.source,
+ sourceMetadata: row.sourceMetadata,
+ mimeType: row.mimeType,
+ bytes: row.bytes,
+ category: 'script' as const,
+ isExecutable: row.isExecutable ?? false,
+ author: new Types.ObjectId(row.author),
+ tenantId: row.tenantId,
+ };
+ files.set(row.relativePath, next);
+ return next;
+ },
+ );
+ const deps = createDeps({
+ findSkillBySourceIdentity: jest.fn(async () => existing),
+ getSkillById: jest.fn(async () => ({ ...existing, version: existing.version + 1 })),
+ getSkillFileByPath: jest.fn(
+ async (_skillId, relativePath) => files.get(relativePath) ?? null,
+ ),
+ listSkillFiles: jest.fn(async () => Array.from(files.values())),
+ upsertSkillFile,
+ deleteSkillFile: jest.fn(async (_skillId, relativePath) => ({
+ deleted: files.delete(relativePath),
+ })),
+ reserveBlob: jest.fn(async ({ userId, fileName, tenantId }) => ({
+ filepath: '/uploads/new-file-id__run.sh',
+ source: 'local',
+ writeLocation: { userId, fileName, basePath: 'uploads' as const, tenantId },
+ })),
+ saveBuffer: jest.fn(async () => ({
+ filepath: '/uploads/new-file-id__run.sh',
+ source: 'local',
+ })),
+ deleteFile: jest.fn(async () => undefined),
+ queueCommittedSkillUploadCleanup: jest.fn(async () => {
+ if (queueFailed) throw new Error('cleanup queue unavailable');
+ }),
+ updateSkill: jest.fn(async () => ({ status: 'conflict' as const, current: existing })),
+ });
+ const runner = createGitHubSkillSyncRunner(deps);
+ const result = await runner.runOnce();
- expect(result.status).toBe('failed');
- expect(deps.upsertStatus).toHaveBeenLastCalledWith(
- expect.objectContaining({
- status: 'failed',
- errorCode: 'SKILL_CONFLICT',
- }),
- );
- expect(files.get('scripts/run.sh')).toEqual(
- expect.objectContaining({ filepath: oldFile.filepath }),
- );
- expect(deps.deleteFile).toHaveBeenCalledWith(
- expect.objectContaining({ filepath: '/uploads/new-file-id__run.sh' }),
- );
- expect(deps.deleteFile).not.toHaveBeenCalledWith(
- expect.objectContaining({ filepath: oldFile.filepath }),
- );
- });
+ expect(result.status).toBe('failed');
+ expect(deps.upsertStatus).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ status: 'failed',
+ errorCode: queueFailed ? 'SYNC_ROLLBACK_FAILED' : 'SKILL_CONFLICT',
+ }),
+ );
+ expect(files.get('scripts/run.sh')).toEqual(
+ expect.objectContaining({
+ filepath: queueFailed ? '/uploads/new-file-id__run.sh' : oldFile.filepath,
+ }),
+ );
+ if (queueFailed) {
+ expect(deps.deleteFile).not.toHaveBeenCalled();
+ expect(deps.upsertStatus).toHaveBeenLastCalledWith(
+ expect.objectContaining({ status: 'failed', errorCode: 'SYNC_ROLLBACK_FAILED' }),
+ );
+ } else {
+ expect(deps.deleteFile).toHaveBeenCalledWith(
+ expect.objectContaining({ filepath: '/uploads/new-file-id__run.sh' }),
+ );
+ }
+ expect(deps.deleteFile).not.toHaveBeenCalledWith(
+ expect.objectContaining({ filepath: oldFile.filepath }),
+ );
+ expect(deps.queueCommittedSkillUploadCleanup).toHaveBeenCalledWith(expect.any(String));
+ expect(deps.markSkillUploadIntentCleaned).toHaveBeenCalledTimes(queueFailed ? 0 : 1);
+ },
+ );
it('fails the source when existing-skill rollback cannot remove replacement storage', async () => {
const existing = makeSkill({
@@ -3093,6 +3130,11 @@ describe('createGitHubSkillSyncRunner', () => {
skillId: row.skillId as Types.ObjectId,
author: new Types.ObjectId(row.author),
})),
+ reserveBlob: jest.fn(async ({ userId, fileName, tenantId }) => ({
+ filepath: '/uploads/new-file-id__run.sh',
+ source: 'local',
+ writeLocation: { userId, fileName, basePath: 'uploads' as const, tenantId },
+ })),
saveBuffer: jest.fn(async () => ({
filepath: '/uploads/new-file-id__run.sh',
source: 'local',
@@ -3107,6 +3149,8 @@ describe('createGitHubSkillSyncRunner', () => {
expect(deleteFile).toHaveBeenCalledWith(
expect.objectContaining({ filepath: '/uploads/new-file-id__run.sh' }),
);
+ expect(deps.queueCommittedSkillUploadCleanup).toHaveBeenCalledWith(expect.any(String));
+ expect(deps.markSkillUploadIntentCleaned).not.toHaveBeenCalled();
expect(deps.upsertStatus).toHaveBeenLastCalledWith(
expect.objectContaining({
status: 'failed',
@@ -3810,6 +3854,28 @@ describe('steady-state sync of an unchanged source', () => {
repo: ReturnType<typeof createRepo>,
sourceOverrides: Partial<{ ref: string }> = {},
) {
+ const deleteSkillWithStorage = async (id: string): Promise<DeleteSkillResult> => {
+ const progress = await runSkillDeletion(id, {
+ ...createSkillDeletionPersistenceAdapter(db),
+ deleteSkillFileAfterBlob: db.deleteSkillFileAfterBlob,
+ finalizeSkillDeletion: db.finalizeSkillDeletion,
+ completeSkillDeletion: async (skillId) => {
+ const owner = 'sync-test-worker';
+ if (!(await db.claimSkillDeletion(skillId, owner, 60_000))) return false;
+ return db.completeSkillDeletion(skillId, owner);
+ },
+ reopenSkillDeletion: db.reopenSkillDeletion,
+ markSkillUploadIntentCleaned: db.markSkillUploadIntentCleaned,
+ deleteBlob: async () => undefined,
+ deleteReservedBlob: async () => undefined,
+ });
+ return {
+ deleted: progress.found && progress.cleanupComplete,
+ skillAbsent: progress.found && progress.cleanupComplete,
+ cleanupComplete: progress.found && progress.cleanupComplete,
+ failedCleanupSteps: progress.failedSteps,
+ };
+ };
return createDeps({
getConfig: () => ({
github: {
@@ -3838,9 +3904,9 @@ describe('steady-state sync of an unchanged source', () => {
getSkillFileByPath: jest.fn(db.getSkillFileByPath),
upsertSkillFile: jest.fn(db.upsertSkillFile),
deleteSkillFile: jest.fn(db.deleteSkillFile),
- deleteSkill: jest.fn(db.deleteSkill),
- saveBuffer: jest.fn(async ({ fileName }: { fileName: string }) => ({
- filepath: `/uploads/${fileName}`,
+ deleteSkill: jest.fn(deleteSkillWithStorage),
+ saveBuffer: jest.fn(async ({ reservation }) => ({
+ filepath: reservation.filepath,
source: 'local',
})),
});
@@ -3970,6 +4036,123 @@ describe('steady-state sync of an unchanged source', () => {
expect(file?.sourceMetadata?.blobSha).toBe('skills/research/scripts/run.sh@2');
});
+ it.each([true, false])(
+ 'does not sweep a superseded blob before the sync %s outcome',
+ async (rollBack) => {
+ const repo = createRepo({
+ 'skills/research/SKILL.md': { content: researchMarkdown },
+ 'skills/research/scripts/run.sh': { content: 'echo old' },
+ });
+ const objects = new Map<string, Buffer>();
+ const withRealStorage = (deps: ReturnType<typeof createDbDeps>) => {
+ deps.reserveBlob = jest.fn(async ({ userId, fileName, tenantId }) => ({
+ filepath: `/uploads/${userId}/${fileName}`,
+ source: 'local',
+ writeLocation: { userId, fileName, basePath: 'uploads' as const, tenantId },
+ }));
+ deps.saveBuffer = jest.fn(async ({ reservation, buffer }) => {
+ objects.set(reservation.filepath, Buffer.from(buffer));
+ return { filepath: reservation.filepath, source: reservation.source };
+ });
+ deps.deleteFile = jest.fn(async (file) => {
+ objects.delete(file.filepath);
+ });
+ deps.reserveSkillUploadIntent = jest.fn(db.reserveSkillUploadIntent);
+ deps.transitionSkillUploadIntent = jest.fn(db.transitionSkillUploadIntent);
+ deps.queueCommittedSkillUploadCleanup = jest.fn(db.queueCommittedSkillUploadCleanup);
+ deps.finishSkillUploadCleanupHold = jest.fn(db.finishSkillUploadCleanupHold);
+ deps.markSkillUploadIntentCleaned = jest.fn(db.markSkillUploadIntentCleaned);
+ return deps;
+ };
+
+ const seeded = await createGitHubSkillSyncRunner(
+ withRealStorage(createDbDeps(repo)),
+ ).runOnce();
+ expect(seeded.status).toBe('completed');
+ const mirror = await getMirror();
+ const oldFile = await db.getSkillFileByPath(mirror._id, 'scripts/run.sh');
+ expect(oldFile).not.toBeNull();
+ expect(objects.get(oldFile!.filepath)?.toString()).toBe('echo old');
+
+ repo.setFiles({
+ 'skills/research/SKILL.md': { content: researchMarkdown },
+ 'skills/research/scripts/run.sh': { content: 'echo new', revision: 2 },
+ });
+ let reachedReplacement!: () => void;
+ const replaced = new Promise<void>((resolve) => {
+ reachedReplacement = resolve;
+ });
+ let resumeSync!: () => void;
+ const resume = new Promise<void>((resolve) => {
+ resumeSync = resolve;
+ });
+ const deps = withRealStorage(createDbDeps(repo));
+ let paused = false;
+ deps.upsertSkillFile = jest.fn(async (row) => {
+ const result = await db.upsertSkillFile(row);
+ if (!paused) {
+ paused = true;
+ reachedReplacement();
+ await resume;
+ }
+ return result;
+ });
+ if (rollBack) {
+ deps.updateSkill = jest.fn(async () => ({ status: 'conflict' as const, current: mirror }));
+ }
+ const sync = createGitHubSkillSyncRunner(deps).runOnce();
+ await replaced;
+
+ let finishSweep!: () => void;
+ const swept = new Promise<void>((resolve) => {
+ finishSweep = resolve;
+ });
+ const deletion = createSkillDeletionService({
+ persistence: { ...db, ...createSkillDeletionPersistenceAdapter(db) },
+ loadConfig: async () =>
+ ({ skillDeletion: { finalizationEnabled: false } }) as unknown as AppConfig,
+ getStrategyFunctions: () => ({
+ deleteFile: async (_req, file) => {
+ objects.delete(file.filepath);
+ },
+ }),
+ runAsSystem: async (callback) => {
+ try {
+ return await callback();
+ } finally {
+ finishSweep();
+ }
+ },
+ logger: { error: jest.fn(), warn: jest.fn() },
+ });
+ const sweep = deletion.startSkillDeletionSweep({ sweepIntervalMs: 600_000 });
+ let oldBlobAfterSweep: string | undefined;
+ try {
+ await swept;
+ oldBlobAfterSweep = objects.get(oldFile!.filepath)?.toString();
+ } finally {
+ sweep.stop();
+ resumeSync();
+ }
+
+ const result = await sync;
+ expect(oldBlobAfterSweep).toBe('echo old');
+ if (rollBack) {
+ expect(result.status).toBe('failed');
+ expect((await db.getSkillFileByPath(mirror._id, 'scripts/run.sh'))?.file_id).toBe(
+ oldFile!.file_id,
+ );
+ expect(objects.get(oldFile!.filepath)?.toString()).toBe('echo old');
+ } else {
+ expect(result.status).toBe('completed');
+ expect((await db.getSkillFileByPath(mirror._id, 'scripts/run.sh'))?.file_id).not.toBe(
+ oldFile!.file_id,
+ );
+ expect(objects.has(oldFile!.filepath)).toBe(false);
+ }
+ },
+ );
+
it('updates the skill and deletes the file when a bundled file is removed upstream', async () => {
const repo = createRepo(baseRepo());
await seed(repo);
diff --git a/packages/api/src/skills/sync/github.ts b/packages/api/src/skills/sync/github.ts
index 11d4da0647..f6b389b795 100644
--- a/packages/api/src/skills/sync/github.ts
+++ b/packages/api/src/skills/sync/github.ts
@@ -33,11 +33,14 @@ import type {
AssertNotCancelled,
} from './adapters/types';
import type { GitHubRepoAdapterConfig } from './adapters/github';
+import type { SkillBlobReservation } from '../reservation';
+import type { SkillUploadIntentDeps } from '../intent';
import {
GITHUB_FINE_GRAINED_TOKEN_RECOMMENDATION,
createGitHubRepoAdapter,
} from './adapters/github';
import { DEFAULT_SKILL_IMPORT_LIMITS } from '../limits';
+import { withSkillUploadIntent } from '../intent';
import { parseSkillMarkdown } from '../parse';
import { normalizeRepoPath } from './path';
import { SkillSyncError } from './errors';
@@ -134,6 +137,7 @@ type SaveBufferResult = {
};
type StoredSkillFileRef = {
+ file_id?: string;
filepath: string;
source: string;
storageKey?: string;
@@ -150,6 +154,7 @@ type DeletedSyncedSkillJournal = {
type SyncSkillFilesJournal = {
staleFiles: StoredSkillFileRef[];
savedFiles: StoredSkillFileRef[];
+ heldSupersededFileIds: string[];
};
type SyncSkillFilesResult = Pick<SyncCounters, 'syncedFileCount' | 'deletedFileCount'> &
@@ -157,7 +162,14 @@ type SyncSkillFilesResult = Pick<SyncCounters, 'syncedFileCount' | 'deletedFileC
type MaybePromise<T> = T | Promise<T>;
-export type GitHubSkillSyncDeps = {
+export type GitHubSkillSyncDeps = SkillUploadIntentDeps & {
+ queueCommittedSkillUploadCleanup: (fileId: string) => Promise<void>;
+ markSkillUploadIntentCleaned: (fileId: string) => Promise<void>;
+ finishSkillUploadCleanupHold: (
+ fileId: string,
+ holdOwner: string,
+ cleanup: boolean,
+ ) => Promise<boolean>;
getConfig: () => MaybePromise<SkillSyncConfig | undefined>;
getCredentialToken: (
provider: SkillSyncProvider,
@@ -217,13 +229,15 @@ export type GitHubSkillSyncDeps = {
) => Promise<{ deleted: boolean }>;
deleteSkill: (id: string) => Promise<DeleteSkillResult>;
saveBuffer: (params: {
- userId: string;
+ reservation: SkillBlobReservation;
buffer: Buffer;
+ }) => Promise<SaveBufferResult>;
+ reserveBlob: (params: {
+ userId: string;
fileName: string;
- basePath?: string;
isImage?: boolean;
tenantId?: string;
- }) => Promise<SaveBufferResult>;
+ }) => Promise<SkillBlobReservation>;
deleteFile?: (file: {
filepath: string;
source: string;
@@ -1106,10 +1120,12 @@ async function cleanupFile(deps: GitHubSkillSyncDeps, file: StoredSkillFileRef):
function toStoredFileRef(params: {
saved: SaveBufferResult;
+ fileId: string;
author: Types.ObjectId;
tenantId?: string;
}): StoredSkillFileRef {
return {
+ file_id: params.fileId,
filepath: params.saved.filepath,
source: params.saved.source,
storageKey: params.saved.storageKey,
@@ -1138,36 +1154,6 @@ function toSkillFileInput(file: ISkillFile & { _id: Types.ObjectId }): UpsertSki
};
}
-function toCreateSkillInput(skill: ISkill & { _id: Types.ObjectId }): CreateSkillInput {
- return {
- name: skill.name,
- displayTitle: skill.displayTitle,
- description: skill.description,
- body: skill.body,
- frontmatter: skill.frontmatter,
- category: skill.category,
- author: skill.author,
- authorName: skill.authorName,
- source: PROVIDER,
- sourceMetadata: skill.sourceMetadata,
- alwaysApply: skill.alwaysApply,
- tenantId: skill.tenantId,
- };
-}
-
-function toStoredFileRefFromSkillFile(
- file: ISkillFile & { _id: Types.ObjectId },
-): StoredSkillFileRef {
- return {
- filepath: file.filepath,
- source: file.source,
- storageKey: file.storageKey,
- storageRegion: file.storageRegion,
- author: file.author,
- tenantId: file.tenantId,
- };
-}
-
function getStoredFileKey(file: StoredSkillFileRef): string {
return [file.source, file.filepath, file.storageKey ?? '', file.storageRegion ?? ''].join(':');
}
@@ -1177,6 +1163,7 @@ async function cleanupStoredFiles(params: {
files: StoredSkillFileRef[];
logMessage: string;
throwOnError?: boolean;
+ onCleaned?: (file: StoredSkillFileRef) => Promise<void>;
}): Promise<void> {
const seen = new Set<string>();
const cleanupErrors: unknown[] = [];
@@ -1186,7 +1173,13 @@ async function cleanupStoredFiles(params: {
continue;
}
seen.add(key);
- await cleanupFile(params.deps, file).catch((cleanupError) => {
+ await (async () => {
+ if (params.onCleaned && !params.deps.deleteFile) {
+ throw new Error('Skill storage has no deletion strategy');
+ }
+ await cleanupFile(params.deps, file);
+ await params.onCleaned?.(file);
+ })().catch((cleanupError) => {
cleanupErrors.push(cleanupError);
logger.error(params.logMessage, cleanupError);
});
@@ -1201,8 +1194,21 @@ async function restoreExistingSkillFiles(params: {
skill: ISkill & { _id: Types.ObjectId };
previousFiles: Array<ISkillFile & { _id: Types.ObjectId }>;
savedFiles: StoredSkillFileRef[];
+ heldSupersededFileIds: string[];
+ cleanupHoldOwner: string;
}): Promise<void> {
- const { deps, skill, previousFiles, savedFiles } = params;
+ const { deps, skill, previousFiles, savedFiles, heldSupersededFileIds, cleanupHoldOwner } =
+ params;
+ // Queue new blobs before restoring old rows so failed provider cleanup is retryable.
+ const queued = await Promise.allSettled(
+ savedFiles
+ .filter((file) => file.file_id)
+ .map((file) => deps.queueCommittedSkillUploadCleanup(file.file_id!)),
+ );
+ const queueFailure = queued.find((result) => result.status === 'rejected');
+ // Restoring old rows would orphan a committed replacement whose cleanup
+ // intent was not durably queued. Leave the new rows intact for a later sync.
+ if (queueFailure?.status === 'rejected') throw queueFailure.reason;
const previousByPath = new Map(previousFiles.map((file) => [file.relativePath, file]));
const currentFiles = await deps.listSkillFiles(skill._id);
@@ -1215,12 +1221,23 @@ async function restoreExistingSkillFiles(params: {
for (const file of previousFiles) {
await deps.upsertSkillFile(toSkillFileInput(file));
}
+ // The old rows are live again before their cleanup holds are removed.
+ // If release fails, retain the durable hold for stopped-writer recovery.
+ for (const fileId of new Set(heldSupersededFileIds)) {
+ await deps.finishSkillUploadCleanupHold(fileId, cleanupHoldOwner, false);
+ }
+ let cleanupFailure: unknown;
await cleanupStoredFiles({
deps,
files: savedFiles,
logMessage: '[GitHubSkillSync] Failed to clean up rolled-back synced file:',
throwOnError: true,
+ onCleaned: (file) =>
+ file.file_id ? deps.markSkillUploadIntentCleaned(file.file_id) : Promise.resolve(),
+ }).catch((error) => {
+ cleanupFailure = error;
});
+ if (cleanupFailure) throw cleanupFailure;
}
async function deleteSyncedSkillForRestore(
@@ -1230,14 +1247,6 @@ async function deleteSyncedSkillForRestore(
const files = await deps.listSkillFiles(skill._id);
const deletion = await deps.deleteSkill(skill._id.toString());
if (!deletion.cleanupComplete) {
- if (!deletion.failedCleanupSteps.includes('skill_files')) {
- await cleanupStoredFiles({
- deps,
- files: files.map(toStoredFileRefFromSkillFile),
- logMessage: '[GitHubSkillSync] Failed to clean up partially deleted stale skill file:',
- throwOnError: true,
- });
- }
throw new Error(
`Skill cleanup did not finish: ${deletion.failedCleanupSteps.join(', ') || 'unknown step'}`,
);
@@ -1248,31 +1257,6 @@ async function deleteSyncedSkillForRestore(
};
}
-async function restoreDeletedSyncedSkill(
- deps: GitHubSkillSyncDeps,
- deleted: DeletedSyncedSkillJournal,
-): Promise<void> {
- const restored = await deps.createSkill(toCreateSkillInput(deleted.skill));
- for (const file of deleted.files) {
- await deps.upsertSkillFile({
- ...toSkillFileInput(file),
- skillId: restored.skill._id,
- });
- }
- await ensurePublicViewer(deps, restored.skill._id);
-}
-
-async function cleanupDeletedSyncedSkillFiles(
- deps: GitHubSkillSyncDeps,
- deleted: DeletedSyncedSkillJournal,
-): Promise<void> {
- await cleanupStoredFiles({
- deps,
- files: deleted.files.map(toStoredFileRefFromSkillFile),
- logMessage: '[GitHubSkillSync] Failed to clean up deleted stale mirrored skill file:',
- });
-}
-
function comparableSourceMetadata(metadata: Record<string, unknown> | undefined): string {
const { commitSha: _commitSha, syncedAt: _syncedAt, ...rest } = metadata ?? {};
return JSON.stringify(rest);
@@ -1471,9 +1455,11 @@ async function syncSkillFiles(params: {
discovered: DiscoveredSkill;
assertNotCancelled: AssertNotCancelled;
journal?: SyncSkillFilesJournal;
+ cleanupHoldOwner?: string;
}): Promise<SyncSkillFilesResult> {
- const { deps, adapter, commit, source, skill, discovered, assertNotCancelled } = params;
- const journal = params.journal ?? { staleFiles: [], savedFiles: [] };
+ const { deps, adapter, commit, source, skill, discovered, assertNotCancelled, cleanupHoldOwner } =
+ params;
+ const journal = params.journal ?? { staleFiles: [], savedFiles: [], heldSupersededFileIds: [] };
const remotePaths = new Set<string>();
let syncedFileCount = 0;
let deletedFileCount = 0;
@@ -1496,48 +1482,85 @@ async function syncSkillFiles(params: {
assertNotCancelled();
assertGitHubBufferSize(buffer, relativePath);
const fileId = crypto.randomUUID();
+ if (existing && cleanupHoldOwner) {
+ // Record before the DB write: even a failed CAS may have acquired the
+ // hold, and rollback must release it only after restoring the old row.
+ journal.heldSupersededFileIds.push(existing.file_id);
+ }
const filename = getFilename(relativePath);
const mimeType = guessMimeType(filename);
- const saved = await deps.saveBuffer({
+ const fileName = `${fileId}__${filename}`;
+ const reserved = await deps.reserveBlob({
userId: skill.author.toString(),
- buffer,
- fileName: `${fileId}__${filename}`,
- basePath: 'uploads',
+ fileName,
isImage: mimeType.startsWith('image/'),
tenantId: skill.tenantId,
});
- const savedFile = toStoredFileRef({ saved, author: skill.author, tenantId: skill.tenantId });
- try {
- await deps.upsertSkillFile({
- skillId: skill._id,
- relativePath,
- file_id: fileId,
- filename,
- filepath: saved.filepath,
- storageKey: saved.storageKey,
- storageRegion: saved.storageRegion,
- source: saved.source,
- sourceMetadata: {
- provider: PROVIDER,
- sourceId: source.id,
- upstreamId: makeUpstreamId(source, discovered.rootPath),
- commitSha: commit.id,
- blobSha: entry.id,
- path: entry.path,
- },
- mimeType,
- bytes: buffer.length,
- isExecutable: false,
- author: skill.author,
- tenantId: skill.tenantId,
- });
- } catch (error) {
- await cleanupFile(deps, savedFile).catch((cleanupError) => {
- logger.error('[GitHubSkillSync] Failed to clean up orphaned synced file:', cleanupError);
- throw makeRollbackFailure(error);
- });
- throw error;
- }
+ if (!deps.deleteFile) throw new Error('Skill storage has no deletion strategy');
+ let saved: SaveBufferResult | undefined;
+ await withSkillUploadIntent(deps, {
+ skillId: skill._id.toString(),
+ author: skill.author.toString(),
+ tenantId: skill.tenantId,
+ fileId,
+ source: reserved.source,
+ reservation: reserved,
+ save: async () => {
+ saved = await deps.saveBuffer({
+ reservation: reserved,
+ buffer,
+ });
+ if (saved.source !== reserved.source) throw new Error('Skill storage changed during sync');
+ return saved.filepath;
+ },
+ persist: async (filepath) => {
+ await deps.upsertSkillFile({
+ skillId: skill._id,
+ relativePath,
+ file_id: fileId,
+ ...(existing && cleanupHoldOwner ? { cleanupHoldOwner } : {}),
+ filename,
+ filepath,
+ storageKey: saved?.storageKey,
+ storageRegion: saved?.storageRegion,
+ source: reserved.source,
+ sourceMetadata: {
+ provider: PROVIDER,
+ sourceId: source.id,
+ upstreamId: makeUpstreamId(source, discovered.rootPath),
+ commitSha: commit.id,
+ blobSha: entry.id,
+ path: entry.path,
+ },
+ mimeType,
+ bytes: buffer.length,
+ isExecutable: false,
+ author: skill.author,
+ tenantId: skill.tenantId,
+ });
+ return fileId;
+ },
+ readPersisted: async () =>
+ (await deps.getSkillFileByPath(skill._id, relativePath))?.file_id === fileId
+ ? fileId
+ : null,
+ deleteReserved: (ref, savedPath) =>
+ deps.deleteFile!({
+ filepath: savedPath ?? ref.filepath,
+ storageKey: saved?.storageKey ?? ref.storageKey,
+ storageRegion: saved?.storageRegion ?? ref.storageRegion,
+ source: reserved.source,
+ user: skill.author,
+ tenantId: skill.tenantId,
+ }),
+ });
+ if (!saved) throw new Error('Skill upload returned no stored file');
+ const savedFile = toStoredFileRef({
+ saved,
+ fileId,
+ author: skill.author,
+ tenantId: skill.tenantId,
+ });
syncedFileCount++;
journal.savedFiles.push(savedFile);
if (existing && existing.filepath !== saved.filepath) {
@@ -1566,26 +1589,12 @@ async function deleteSyncedSkill(
): Promise<number> {
const files = await deps.listSkillFiles(skill._id);
const deletion = await deps.deleteSkill(skill._id.toString());
- let deletedFiles = 0;
- const cleanupErrors: unknown[] = [];
- if (!deletion.failedCleanupSteps.includes('skill_files')) {
- for (const file of files) {
- await cleanupFile(deps, file).catch((cleanupError) => {
- cleanupErrors.push(cleanupError);
- logger.error('[GitHubSkillSync] Failed to clean up mirrored skill file:', cleanupError);
- });
- deletedFiles++;
- }
- }
if (!deletion.cleanupComplete) {
throw new Error(
`Skill cleanup did not finish: ${deletion.failedCleanupSteps.join(', ') || 'unknown step'}`,
);
}
- if (cleanupErrors.length > 0) {
- throw cleanupErrors[0];
- }
- return deletedFiles;
+ return files.length;
}
function getTokenEnvVarName(tokenReference: string | undefined): string | null {
@@ -1892,9 +1901,6 @@ async function syncSource(params: {
existingSyncedSkills = staleConflictCleanup.remainingSkills;
counts.deletedSkillCount += staleConflictCleanup.deletedSkillCount;
counts.deletedFileCount += staleConflictCleanup.deletedFileCount;
- if (staleConflictCleanup.deletedSkill) {
- await cleanupDeletedSyncedSkillFiles(deps, staleConflictCleanup.deletedSkill);
- }
}
counts.syncedSkillCount++;
recordUnsupportedFiles(discovered);
@@ -1947,7 +1953,12 @@ async function syncSource(params: {
}
await ensurePublicViewer(deps, effectivePrepared.existing._id);
const previousFiles = await deps.listSkillFiles(effectivePrepared.existing._id);
- const journal: SyncSkillFilesJournal = { staleFiles: [], savedFiles: [] };
+ const cleanupHoldOwner = crypto.randomUUID();
+ const journal: SyncSkillFilesJournal = {
+ staleFiles: [],
+ savedFiles: [],
+ heldSupersededFileIds: [],
+ };
let fileCounts: SyncSkillFilesResult;
let staleConflictCleanup:
| Awaited<ReturnType<typeof deleteNameConflictingStaleSkill>>
@@ -1962,6 +1973,7 @@ async function syncSource(params: {
discovered,
assertNotCancelled,
journal,
+ cleanupHoldOwner,
});
if (prepared.existing && canReconcileStaleSkills) {
staleConflictCleanup = await deleteNameConflictingStaleSkill({
@@ -1994,6 +2006,8 @@ async function syncSource(params: {
skill: effectivePrepared.existing,
previousFiles,
savedFiles: journal.savedFiles,
+ heldSupersededFileIds: journal.heldSupersededFileIds,
+ cleanupHoldOwner,
}).catch((cleanupError) => {
rollbackFailed = true;
logger.error(
@@ -2002,18 +2016,9 @@ async function syncSource(params: {
);
});
if (staleConflictCleanup?.deletedSkill) {
- await restoreDeletedSyncedSkill(deps, staleConflictCleanup.deletedSkill).catch(
- (cleanupError) => {
- logger.error(
- '[GitHubSkillSync] Failed to recreate stale mirrored skill after sync failure:',
- cleanupError,
- );
- },
- );
- /* deleteSkill removes the original id from agent allowlists and
- deletes every ACL entry. Recreating the row recovers its data,
- but cannot restore that dependent state, so this is never a
- complete rollback and the source must fail visibly. */
+ /* The old mirror's blobs and permissions are gone. A replacement
+ row pointing at those deleted blobs would be corrupt; the next
+ source run must rebuild from upstream instead. */
rollbackFailed = true;
}
throw rollbackFailed ? makeRollbackFailure(error) : error;
@@ -2022,9 +2027,14 @@ async function syncSource(params: {
deps,
files: fileCounts.staleFiles,
logMessage: '[GitHubSkillSync] Failed to clean up replaced synced file:',
+ onCleaned: (file) =>
+ file.file_id ? deps.markSkillUploadIntentCleaned(file.file_id) : Promise.resolve(),
});
- if (staleConflictCleanup?.deletedSkill) {
- await cleanupDeletedSyncedSkillFiles(deps, staleConflictCleanup.deletedSkill);
+ // Metadata has committed: do not roll back if cleanup release fails.
+ // Retained holds are safe and can be recovered after the writer stops.
+ for (const fileId of new Set(journal.heldSupersededFileIds)) {
+ const released = await deps.finishSkillUploadCleanupHold(fileId, cleanupHoldOwner, true);
+ if (!released) throw new Error(`Skill cleanup hold was lost for ${fileId}`);
}
counts.syncedSkillCount++;
counts.syncedFileCount += fileCounts.syncedFileCount;
diff --git a/packages/api/src/skills/upload.spec.ts b/packages/api/src/skills/upload.spec.ts
index 9d5f0b2e86..bef0ff8471 100644
--- a/packages/api/src/skills/upload.spec.ts
+++ b/packages/api/src/skills/upload.spec.ts
@@ -1,3 +1,4 @@
+import { Types } from 'mongoose';
import { FileSources } from 'librechat-data-provider';
import type { Response } from 'express';
import { createSkillUploadHandler } from './upload';
@@ -25,13 +26,25 @@ const originalFile = {
function harness() {
const skill = deferred<Awaited<ReturnType<UploadDeps['getSkillById']>>>();
const file = deferred<Awaited<ReturnType<UploadDeps['getSkillFileByPath']>>>();
- const saveBuffer = jest.fn(async () => '/uploads/replacement');
+ const saveBuffer = jest.fn(async ({ userId, fileName }) => `/uploads/${userId}/${fileName}`);
+ const deleteFile = jest.fn(async () => undefined);
const deps = {
getSkillById: jest.fn(() => skill.promise),
getSkillFileByPath: jest.fn(() => file.promise),
- upsertSkillFile: jest.fn(async () => ({ ...originalFile, file_id: 'saved' })),
+ upsertSkillFile: jest.fn(async (row) => ({
+ ...originalFile,
+ file_id: row.file_id,
+ filepath: row.filepath,
+ })),
resolveStorage: jest.fn(() => ({ source: FileSources.local, saveBuffer })),
- getStrategyFunctions: jest.fn(() => ({})),
+ getStrategyFunctions: jest.fn(() => ({ deleteFile })),
+ reserveSkillUploadIntent: jest.fn(async (input) => ({
+ ...input,
+ skillId: new Types.ObjectId(),
+ author: new Types.ObjectId(),
+ state: 'uploading' as const,
+ })),
+ transitionSkillUploadIntent: jest.fn(async () => true),
} satisfies UploadDeps;
const req = {
params: { id: 'skill-id', relativePath: ['references', 'guide.md'] },
@@ -47,10 +60,31 @@ function harness() {
const res = {} as Response;
res.status = jest.fn(() => res);
res.json = jest.fn(() => res);
- return { skill, file, deps, saveBuffer, req, res, handler: createSkillUploadHandler(deps) };
+ return {
+ skill,
+ file,
+ deps,
+ saveBuffer,
+ deleteFile,
+ req,
+ res,
+ handler: createSkillUploadHandler(deps),
+ };
}
describe('skill upload lookup ordering', () => {
+ it('leaves old-blob cleanup to the durable persistence queue after browser replacement', async () => {
+ const h = harness();
+ const pending = h.handler(h.req, h.res);
+ h.skill.resolve({ source: 'inline' });
+ h.file.resolve(originalFile);
+ await pending;
+ expect(h.res.status).toHaveBeenCalledWith(200);
+ expect(h.deps.upsertSkillFile).toHaveBeenCalledWith(
+ expect.objectContaining({ expectedFileId: 'original' }),
+ );
+ expect(h.deleteFile).not.toHaveBeenCalled();
+ });
it.each(['skill', 'file'] as const)(
'starts both privileged lookups before either resolves (%s first)',
async (first) => {
@@ -91,6 +125,25 @@ describe('skill upload lookup ordering', () => {
expect(h.res.status).toHaveBeenCalledWith(200);
});
+ it.each(['reservation', 'persistence'] as const)(
+ 'reports a deletion race during %s as a conflict',
+ async (stage) => {
+ const h = harness();
+ const error = Object.assign(new Error('Internal deletion state'), { code: 'SKILL_DELETING' });
+ if (stage === 'reservation') h.deps.reserveSkillUploadIntent.mockRejectedValueOnce(error);
+ else h.deps.upsertSkillFile.mockRejectedValueOnce(error);
+
+ const pending = h.handler(h.req, h.res);
+ h.skill.resolve({ source: 'inline' });
+ h.file.resolve(originalFile);
+ await pending;
+
+ expect(h.res.status).toHaveBeenCalledWith(409);
+ expect(h.res.json).toHaveBeenCalledWith({ error: 'SKILL_DELETING' });
+ if (stage === 'reservation') expect(h.saveBuffer).not.toHaveBeenCalled();
+ },
+ );
+
it.each(['missing', 'managed', 'conflict', 'lookup failure'] as const)(
'does not store bytes for %s',
async (scenario) => {
diff --git a/packages/api/src/skills/upload.ts b/packages/api/src/skills/upload.ts
index 7b15ac40cd..12446a32f8 100644
--- a/packages/api/src/skills/upload.ts
+++ b/packages/api/src/skills/upload.ts
@@ -5,10 +5,14 @@ import type { TSkillFile, TSkill } from 'librechat-data-provider';
import type { Response } from 'express';
import type { StrategyFunctions, ServerRequest } from '~/types';
import type { SaveBufferParams } from '~/storage/types';
+import type { SkillUploadIntentDeps } from './intent';
import { resolveRequestTenantId } from '~/middleware/tenant';
import { getStorageMetadata } from '~/storage/metadata';
+import { getSafeErrorMetadata } from '~/utils/errors';
import { blockFilteredSkillFile } from './protection';
import { resolveSkillFilePathParam } from './path';
+import { withSkillUploadIntent } from './intent';
+import { reserveSkillBlob } from './reservation';
/** Metadata required for replacement and cleanup, independent of a database document. */
export interface StoredSkillFile {
@@ -22,7 +26,7 @@ export interface StoredSkillFile {
isExecutable?: boolean;
}
-interface SkillUploadDeps {
+interface SkillUploadDeps extends SkillUploadIntentDeps {
getSkillFileByPath: (skillId: string, relativePath: string) => Promise<StoredSkillFile | null>;
getSkillById: (id: string) => Promise<Pick<TSkill, 'source'> | null>;
upsertSkillFile: (
@@ -104,77 +108,80 @@ export function createSkillUploadHandler(
const tenantId = resolveRequestTenantId(req);
const storage = deps.resolveStorage(req, { isImage: file.mimetype.startsWith('image/') });
const fileId = randomUUID();
- const filepath = await storage.saveBuffer({
+ const fileName = `${fileId}__${file.originalname}`;
+ const strategy = deps.getStrategyFunctions(storage.source) as Partial<StrategyFunctions> & {
+ getFileURL?: (params: {
+ userId: string;
+ fileName: string;
+ basePath: string;
+ tenantId?: string;
+ }) => Promise<string>;
+ };
+ if (!strategy.deleteFile) throw new Error('Skill storage has no deletion strategy');
+ const reservation = await reserveSkillBlob({
+ source: storage.source,
userId,
- buffer: file.buffer,
- fileName: `${fileId}__${file.originalname}`,
- basePath: 'uploads',
+ fileName,
tenantId,
+ getFileURL: strategy.getFileURL,
});
- const cleanupReplacedBlob = (): void => {
- if (!existingFile || existingFile.filepath === filepath) {
- return;
- }
- const deleteFile = deps.getStrategyFunctions(existingFile.source).deleteFile;
- if (deleteFile) {
- deleteFile(req, {
- filepath: existingFile.filepath,
- storageKey: existingFile.storageKey,
- storageRegion: existingFile.storageRegion,
- user: existingFile.author?.toString() ?? userId,
- tenantId: existingFile.tenantId ?? tenantId,
- }).catch((error: Error) => logger.error('[uploadFile] Old blob cleanup failed:', error));
- }
- };
- let result: StoredSkillFile;
- try {
- result = await deps.upsertSkillFile({
- skillId,
- relativePath,
- expectedFileId,
- file_id: fileId,
- filename: file.originalname,
- filepath,
- ...getStorageMetadata({ filepath, source: storage.source }),
- source: storage.source,
- mimeType: file.mimetype || 'application/octet-stream',
- bytes: file.size,
- isExecutable: existingFile?.isExecutable ?? false,
- author: userId,
- tenantId,
- });
- } catch (error) {
- try {
- // A parent-version update can fail after the file row committed. Never
- // delete its live blob on that ambiguous failure; retain it for reread.
+ const result = await withSkillUploadIntent(deps, {
+ skillId,
+ author: userId,
+ tenantId,
+ fileId,
+ source: storage.source,
+ reservation,
+ save: () =>
+ storage.saveBuffer({
+ ...reservation.writeLocation,
+ buffer: file.buffer,
+ }),
+ persist: (filepath) =>
+ deps.upsertSkillFile({
+ skillId,
+ relativePath,
+ expectedFileId,
+ file_id: fileId,
+ filename: file.originalname,
+ filepath,
+ ...getStorageMetadata({ filepath, source: storage.source }),
+ source: storage.source,
+ mimeType: file.mimetype || 'application/octet-stream',
+ bytes: file.size,
+ isExecutable: existingFile?.isExecutable ?? false,
+ author: userId,
+ tenantId,
+ }),
+ readPersisted: async () => {
const persisted = await deps.getSkillFileByPath(skillId, relativePath);
- if (persisted?.file_id === fileId) {
- cleanupReplacedBlob();
- } else {
- await deps.getStrategyFunctions(storage.source).deleteFile?.(req, {
- filepath,
- ...getStorageMetadata({ filepath, source: storage.source }),
- user: userId,
- tenantId,
- });
- }
- } catch (cleanupError) {
- logger.error('[uploadFile] Failed to clean up orphaned blob:', cleanupError);
- }
- throw error;
- }
- cleanupReplacedBlob();
+ return persisted?.file_id === fileId ? persisted : null;
+ },
+ deleteReserved: (ref, savedPath) =>
+ strategy.deleteFile!(req, {
+ filepath: savedPath ?? ref.filepath,
+ ...getStorageMetadata({
+ filepath: savedPath ?? ref.filepath,
+ source: storage.source,
+ }),
+ user: userId,
+ tenantId,
+ }),
+ });
return res.status(200).json(result);
} catch (error) {
if (error instanceof Error && 'code' in error) {
if (error.code === 'SKILL_FILE_CONFLICT') {
return res.status(409).json({ error: 'SKILL_FILE_CONFLICT' });
}
+ if (error.code === 'SKILL_DELETING') {
+ return res.status(409).json({ error: 'SKILL_DELETING' });
+ }
if (error.code === 'SKILL_FILE_VALIDATION_FAILED') {
return res.status(400).json({ error: error.message });
}
}
- logger.error('[uploadFile] Error:', error);
+ logger.error('[uploadFile] Error:', getSafeErrorMetadata(error));
return res.status(500).json({ error: 'Failed to upload file' });
}
};
diff --git a/packages/api/src/user/erasure.spec.ts b/packages/api/src/user/erasure.spec.ts
index 9384cf9932..399a989411 100644
--- a/packages/api/src/user/erasure.spec.ts
+++ b/packages/api/src/user/erasure.spec.ts
@@ -1,10 +1,17 @@
import mongoose from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { ResourceType, PrincipalType } from 'librechat-data-provider';
-import { createModels, createMethods, tenantStorage } from '@librechat/data-schemas';
+import {
+ createModels,
+ createMethods,
+ tenantStorage,
+ createSkillDeletionPersistenceAdapter,
+} from '@librechat/data-schemas';
+import type { AppConfig } from '@librechat/data-schemas';
import type { CheckpointDeletion } from '~/agents/checkpoints/deletion';
import type { UserErasureDeps } from './erasure';
import { deleteAllSharedLinksWithCleanup } from '~/shared-links/service';
+import { createSkillDeletionService } from '../skills/deletionService';
import { eraseUserData } from './erasure';
const OWNER_BITS = 15;
@@ -38,7 +45,14 @@ async function seedUser(userId: mongoose.Types.ObjectId) {
const legacyServerId = new mongoose.Types.ObjectId();
await model('Skill').insertOne({ _id: skillId, name: `skill-${user}`, author: userId });
- await model('SkillFile').insertOne({ skillId, relativePath: 'notes.md', author: userId });
+ await model('SkillFile').insertOne({
+ skillId,
+ relativePath: 'notes.md',
+ file_id: `skill-file-${user}`,
+ filepath: `/uploads/${user}/notes.md`,
+ source: 'local',
+ author: userId,
+ });
await methods.grantPermission(
PrincipalType.USER,
userId,
@@ -109,6 +123,30 @@ function createDeps() {
return { deps, checkpointDeletion, order };
}
+async function finishSkillCleanup(skillId: mongoose.Types.ObjectId) {
+ const deleteFile = jest.fn(async () => undefined);
+ const worker = createSkillDeletionService({
+ persistence: { ...methods, ...createSkillDeletionPersistenceAdapter(methods) },
+ loadConfig: async () =>
+ ({ skillDeletion: { finalizationEnabled: true } }) as unknown as AppConfig,
+ getStrategyFunctions: () => ({ deleteFile }),
+ runAsSystem: async (callback) => callback(),
+ logger: { error: jest.fn(), warn: jest.fn() },
+ });
+ const sweep = worker.startSkillDeletionSweep({ sweepIntervalMs: 1_000 });
+ try {
+ for (let attempt = 0; attempt < 50; attempt++) {
+ if ((await count('Skill', { _id: skillId })) === 0) break;
+ await new Promise((resolve) => setTimeout(resolve, 100));
+ }
+ expect(await count('Skill', { _id: skillId })).toBe(0);
+ expect(await count('SkillFile', { skillId })).toBe(0);
+ expect(deleteFile).toHaveBeenCalledTimes(1);
+ } finally {
+ sweep.stop();
+ }
+}
+
describe('eraseUserData', () => {
it('clears skills, MCP servers, shared links, file objects and checkpoints', async () => {
const userId = new mongoose.Types.ObjectId();
@@ -135,8 +173,9 @@ describe('eraseUserData', () => {
]);
expect(await count('File', { user: userId })).toBe(0);
- expect(await count('Skill', { _id: skillId })).toBe(0);
- expect(await count('SkillFile', { skillId })).toBe(0);
+ expect(await methods.getSkillById(skillId)).toBeNull();
+ expect(await count('Skill', { _id: skillId, deletionNeedsJournal: true })).toBe(1);
+ expect(await count('SkillFile', { skillId })).toBe(1);
expect(await count('MCPServer', { author: userId })).toBe(0);
expect(deps.invalidateMCPServerTools).toHaveBeenCalledWith(`acl-${user}`);
expect(deps.invalidateMCPServerTools).toHaveBeenCalledWith(`legacy-${user}`);
@@ -144,6 +183,7 @@ describe('eraseUserData', () => {
expect(deps.disconnectMCPServer).toHaveBeenCalledWith(`legacy-${user}`);
expect(await count('SharedLink', { user })).toBe(0);
expect(await count('AclEntry', { principalId: userId })).toBe(0);
+ await finishSkillCleanup(skillId);
expect(await count('Transaction', { user: userId })).toBe(0);
expect(await count('ToolFavorite', { user: userId })).toBe(0);
expect(await count('ChatProject', { user })).toBe(0);
@@ -414,14 +454,16 @@ describe('eraseUserData', () => {
it('still erases the account when the storage provider fails', async () => {
const userId = new mongoose.Types.ObjectId();
- await seedUser(userId);
+ const { skillId } = await seedUser(userId);
const { deps } = createDeps();
(deps.deleteStoredFiles as jest.Mock).mockRejectedValueOnce(new Error('provider down'));
await eraseUserData({ userId: userId.toString() }, deps);
expect(await count('File', { user: userId })).toBe(0);
- expect(await count('Skill', { author: userId })).toBe(0);
+ expect(await methods.getSkillById(skillId)).toBeNull();
+ expect(await count('AclEntry', { principalId: userId })).toBe(0);
+ await finishSkillCleanup(skillId);
});
for (const modelName of ['Agent', 'PromptGroup']) {
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index f75d81fc37..403b94976c 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -45,6 +45,7 @@ export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX = 64 * 1024;
/** Coalesces a conversation's ready background results into one wake-up turn. */
export const AGENT_BACKGROUND_COMPLETION_RECEIPT_BATCHING_DEFAULT: boolean = true;
export const AGENT_BACKGROUND_SHUTDOWN_INTERRUPT_GRACE_MS_DEFAULT = 5_000;
+export const DEFAULT_SKILL_DELETION_BATCH_SIZE = 100;
import {
DEFAULT_MCP_APP_CSP_LIMITS,
resolveMCPAppCspLimits,
@@ -4086,7 +4087,8 @@ export const configSchema = z.object({
finalizationEnabled: z.boolean().default(false),
sweepIntervalMs: z.number().int().min(1_000).default(60_000),
claimLeaseMs: z.number().int().min(1_000).default(120_000),
- batchSize: z.number().int().min(1).max(1_000).default(100),
+ /** Bounds both sweep candidates and per-Skill manifest batches. */
+ batchSize: z.number().int().min(1).max(1_000).default(DEFAULT_SKILL_DELETION_BATCH_SIZE),
recheckIntervalMs: z.number().int().min(60_000).default(3_600_000),
})
.strict()
@@ -4094,7 +4096,7 @@ export const configSchema = z.object({
finalizationEnabled: false,
sweepIntervalMs: 60_000,
claimLeaseMs: 120_000,
- batchSize: 100,
+ batchSize: DEFAULT_SKILL_DELETION_BATCH_SIZE,
recheckIntervalMs: 3_600_000,
})),
secureImageLinks: z.boolean().optional(),
diff --git a/packages/data-schemas/src/methods/skill.spec.ts b/packages/data-schemas/src/methods/skill.spec.ts
index ee752a3fa1..937011a4b4 100644
--- a/packages/data-schemas/src/methods/skill.spec.ts
+++ b/packages/data-schemas/src/methods/skill.spec.ts
@@ -22,6 +22,7 @@ import {
filterExistingSkillIds,
deriveStructuredFrontmatterFields,
} from './skill';
+import { createSkillLifecycleMethods } from './skillLifecycle';
import { createAclEntryMethods } from './aclEntry';
import { createMethods } from './index';
@@ -111,6 +112,7 @@ afterAll(async () => {
afterEach(async () => {
await Skill.deleteMany({});
await SkillFile.deleteMany({});
+ await mongoose.models.SkillUploadIntent.deleteMany({});
await AclEntry.deleteMany({});
await mongoose.models.Agent.deleteMany({});
});
@@ -775,7 +777,20 @@ describe('Skill CRUD methods', () => {
}
});
- it('deleteSkill cascades ACL entries and skill files', async () => {
+ it('does not expose a deleting Skill as the current version after a missed update', async () => {
+ const { skill } = await methods.createSkill(makeSkillInput());
+ await mongoose.models.Skill.updateOne({ _id: skill._id }, { $set: { deletingAt: new Date() } });
+
+ await expect(
+ methods.updateSkill({
+ id: skill._id.toString(),
+ expectedVersion: skill.version,
+ update: { description: 'Updated description (long enough).' },
+ }),
+ ).resolves.toEqual({ status: 'not_found' });
+ });
+
+ it('queues blob cleanup before finalizing ACL and file rows', async () => {
const { skill } = await methods.createSkill(makeSkillInput());
await grantOwner(skill._id);
await methods.upsertSkillFile({
@@ -794,7 +809,12 @@ describe('Skill CRUD methods', () => {
expect(await SkillFile.countDocuments({ skillId: skill._id })).toBe(1);
const res = await methods.deleteSkill(skill._id.toString());
- expect(res.deleted).toBe(true);
+ expect(res.cleanupComplete).toBe(false);
+ expect(await AclEntry.countDocuments({ resourceId: skill._id })).toBe(1);
+ expect(await SkillFile.countDocuments({ skillId: skill._id })).toBe(1);
+ expect((await methods.finalizeSkillDeletion(skill._id.toString())).cleanupComplete).toBe(false);
+ await methods.deleteSkillFileAfterBlob(skill._id.toString(), 'file-123');
+ expect((await methods.finalizeSkillDeletion(skill._id.toString())).cleanupComplete).toBe(true);
expect(await AclEntry.countDocuments({ resourceId: skill._id })).toBe(0);
expect(await SkillFile.countDocuments({ skillId: skill._id })).toBe(0);
});
@@ -807,40 +827,30 @@ describe('Skill CRUD methods', () => {
expect(filtered).toEqual([canonical]);
});
- it('deleteSkill remains retryable after skill file cleanup fails', async () => {
+ it('does not admit a Skill already fenced for deletion into a new agent allowlist', async () => {
+ const { skill } = await methods.createSkill(makeSkillInput({ name: 'deleting-allowlist' }));
+ await Skill.updateOne({ _id: skill._id }, { $set: { deletingAt: new Date() } });
+ expect(await filterExistingSkillIds(mongoose, [skill._id.toString()])).toEqual([]);
+ });
+
+ it('deleteSkill retains its retry journal instead of removing file rows', async () => {
const { skill } = await methods.createSkill(makeSkillInput({ name: 'flaky-files' }));
const Agent = mongoose.models.Agent;
const agent = await Agent.create(makeAgentDoc([skill._id.toString()]));
- const deleteManySpy = jest
- .spyOn(SkillFile, 'deleteMany')
- .mockRejectedValueOnce(new Error('transient storage failure'));
- try {
- const first = await methods.deleteSkill(skill._id.toString());
- expect(first).toEqual({
- deleted: true,
- skillAbsent: true,
- cleanupComplete: false,
- failedCleanupSteps: ['skill_files'],
- });
-
- const retry = await methods.deleteSkill(skill._id.toString());
- expect(retry).toEqual({
- deleted: false,
- skillAbsent: true,
- cleanupComplete: true,
- failedCleanupSteps: [],
- });
- } finally {
- deleteManySpy.mockRestore();
- }
+ const first = await methods.deleteSkill(skill._id.toString());
+ expect(first).toMatchObject({ cleanupComplete: false, failedCleanupSteps: ['skill_files'] });
+ const retry = await methods.deleteSkill(skill._id.toString());
+ expect(retry).toMatchObject({ cleanupComplete: false, failedCleanupSteps: ['skill_files'] });
+ expect(await Skill.countDocuments({ _id: skill._id })).toBe(1);
+ expect(await mongoose.models.SkillDeletion.countDocuments({ skillId: skill._id })).toBe(1);
const agentAfter = (await Agent.findById(agent._id).lean()) as {
skills?: string[];
skills_enabled?: boolean;
} | null;
- expect(agentAfter?.skills).toEqual([]);
- expect(agentAfter?.skills_enabled).toBe(false);
+ expect(agentAfter?.skills).toEqual([skill._id.toString()]);
+ expect(agentAfter?.skills_enabled).toBe(true);
});
it('deleteSkill disables skills when the deleted id was the entire allowlist', async () => {
@@ -849,7 +859,8 @@ describe('Skill CRUD methods', () => {
const agent = await Agent.create(makeAgentDoc([skill._id.toString()]));
const res = await methods.deleteSkill(skill._id.toString().toUpperCase());
- expect(res.deleted).toBe(true);
+ expect(res.cleanupComplete).toBe(false);
+ expect((await methods.finalizeSkillDeletion(skill._id.toString())).cleanupComplete).toBe(true);
const agentAfter = (await Agent.findById(agent._id).lean()) as {
skills?: string[];
@@ -865,7 +876,8 @@ describe('Skill CRUD methods', () => {
const agent = await Agent.create(makeAgentDoc([skill._id.toString()], { skills_scope: 'all' }));
const res = await methods.deleteSkill(skill._id.toString());
- expect(res.deleted).toBe(true);
+ expect(res.cleanupComplete).toBe(false);
+ expect((await methods.finalizeSkillDeletion(skill._id.toString())).cleanupComplete).toBe(true);
const agentAfter = (await Agent.findById(agent._id).lean()) as {
skills?: string[];
@@ -887,7 +899,8 @@ describe('Skill CRUD methods', () => {
);
const res = await methods.deleteSkill(skill._id.toString());
- expect(res.deleted).toBe(true);
+ expect(res.cleanupComplete).toBe(false);
+ expect((await methods.finalizeSkillDeletion(skill._id.toString())).cleanupComplete).toBe(true);
const agentAfter = (await Agent.findById(agent._id).lean()) as {
skills?: string[];
@@ -911,7 +924,8 @@ describe('Skill CRUD methods', () => {
);
const res = await methods.deleteSkill(skill._id.toString());
- expect(res.deleted).toBe(true);
+ expect(res.cleanupComplete).toBe(false);
+ expect((await methods.finalizeSkillDeletion(skill._id.toString())).cleanupComplete).toBe(true);
const agentAfter = (await Agent.findById(agent._id).lean()) as {
skills?: string[];
@@ -933,7 +947,8 @@ describe('Skill CRUD methods', () => {
);
const res = await methods.deleteSkill(doomed._id.toString());
- expect(res.deleted).toBe(true);
+ expect(res.cleanupComplete).toBe(false);
+ expect((await methods.finalizeSkillDeletion(doomed._id.toString())).cleanupComplete).toBe(true);
const scopedAfter = (await Agent.findById(scoped._id).lean()) as {
skills?: string[];
@@ -1862,6 +1877,7 @@ describe('Skill CRUD methods', () => {
describe('SkillFile methods', () => {
it('allows exactly one conditional writer and rejects a stale or deleted revision', async () => {
+ await mongoose.models.SkillUploadIntent.init();
const { skill } = await methods.createSkill(makeSkillInput());
const input = {
skillId: skill._id,
@@ -1886,6 +1902,18 @@ describe('SkillFile methods', () => {
});
const current = await methods.getSkillFileByPath(skill._id, input.relativePath);
expect(['a', 'b']).toContain(current?.file_id);
+ expect(
+ await mongoose.models.SkillUploadIntent.findOne({ fileId: 'original' }).lean(),
+ ).toMatchObject({
+ state: 'committed',
+ storageKey: '/tmp/original',
+ cleanupRequestedAt: expect.any(Date),
+ });
+ expect(
+ (await createSkillLifecycleMethods(mongoose).listAbandonedSkillUploadIntents()).filter(
+ (intent) => intent.fileId === 'original',
+ ),
+ ).toHaveLength(1);
expect(await methods.getSkillById(skill._id)).toMatchObject({ version: 3, fileCount: 1 });
// An unconditional authoring/agent write also invalidates a captured editor revision.
@@ -1908,6 +1936,145 @@ describe('SkillFile methods', () => {
expect(await methods.getSkillFileByPath(skill._id, input.relativePath)).toBeNull();
});
+ it('keeps a queued blob safe when replacement CAS fails, then retries it after replacement', async () => {
+ const { skill } = await methods.createSkill(makeSkillInput());
+ const input = {
+ skillId: skill._id,
+ relativePath: 'references/queued.md',
+ file_id: 'legacy-original',
+ filename: 'queued.md',
+ filepath: '/tmp/legacy-original',
+ source: 'local',
+ mimeType: 'text/markdown',
+ bytes: 10,
+ author: owner._id,
+ };
+ await methods.upsertSkillFile(input);
+ const spy = jest.spyOn(SkillFile, 'findOneAndUpdate').mockImplementationOnce(
+ () =>
+ ({
+ lean: async () => {
+ throw new Error('CAS unavailable');
+ },
+ }) as never,
+ );
+ try {
+ await expect(
+ methods.upsertSkillFile({
+ ...input,
+ expectedFileId: input.file_id,
+ file_id: 'replacement',
+ }),
+ ).rejects.toThrow('CAS unavailable');
+ } finally {
+ spy.mockRestore();
+ }
+ const lifecycle = createSkillLifecycleMethods(mongoose);
+ expect(await lifecycle.skillFileExistsForIntent(skill._id.toString(), input.file_id)).toBe(
+ true,
+ );
+ const queued = await lifecycle.listAbandonedSkillUploadIntents();
+ expect(queued).toEqual([expect.objectContaining({ fileId: input.file_id })]);
+ const ownerId = 'cleanup-worker';
+ expect(await lifecycle.claimAbandonedSkillUploadIntent(input.file_id, ownerId, 120_000)).toBe(
+ true,
+ );
+ await expect(
+ methods.upsertSkillFile({
+ ...input,
+ expectedFileId: input.file_id,
+ file_id: 'held-replacement',
+ cleanupHoldOwner: 'sync-hold',
+ }),
+ ).rejects.toMatchObject({ code: 'SKILL_FILE_CONFLICT' });
+ expect((await methods.getSkillFileByPath(skill._id, input.relativePath))?.file_id).toBe(
+ input.file_id,
+ );
+ expect(
+ await lifecycle.scheduleAbandonedSkillUploadIntentRecheck(input.file_id, ownerId, 3_600_000),
+ ).toBe(true);
+ await lifecycle.releaseAbandonedSkillUploadIntentClaim(input.file_id, ownerId);
+ expect(await lifecycle.listAbandonedSkillUploadIntents()).toHaveLength(0);
+ await methods.upsertSkillFile({
+ ...input,
+ expectedFileId: input.file_id,
+ file_id: 'replacement',
+ });
+ expect(await lifecycle.skillFileExistsForIntent(skill._id.toString(), input.file_id)).toBe(
+ false,
+ );
+ expect(await lifecycle.listAbandonedSkillUploadIntents()).toEqual([
+ expect.objectContaining({ fileId: input.file_id }),
+ ]);
+ expect(
+ await mongoose.models.SkillUploadIntent.findOne({ fileId: input.file_id }).lean(),
+ ).toMatchObject({
+ storageKey: input.filepath,
+ cleanupRequestedAt: expect.any(Date),
+ });
+ await lifecycle.beginSkillDeletion(skill._id.toString());
+ await lifecycle.beginSkillDeletion(skill._id.toString());
+ expect(await mongoose.models.SkillUploadIntent.countDocuments({ skillId: skill._id })).toBe(2);
+ expect(
+ await mongoose.models.SkillUploadIntent.findOne({ fileId: input.file_id }).lean(),
+ ).toMatchObject({
+ cleanupRequestedAt: expect.any(Date),
+ });
+ });
+
+ it('records the storage key rather than a legacy URL for replacement cleanup', async () => {
+ const { skill } = await methods.createSkill(makeSkillInput());
+ const input = {
+ skillId: skill._id,
+ relativePath: 'references/legacy-cloud.md',
+ file_id: 'legacy-cloud-original',
+ filename: 'legacy-cloud.md',
+ filepath: 'https://stale.example.invalid/legacy-cloud.md',
+ storageKey: 'eu-west-1/skills/legacy-cloud-original.md',
+ storageRegion: 'eu-west-1',
+ source: 's3',
+ mimeType: 'text/markdown',
+ bytes: 10,
+ author: owner._id,
+ };
+ await methods.upsertSkillFile(input);
+ await methods.upsertSkillFile({
+ ...input,
+ file_id: 'legacy-cloud-replacement',
+ filepath: 'https://current.example.invalid/legacy-cloud.md',
+ storageKey: 'eu-west-1/skills/legacy-cloud-replacement.md',
+ });
+ expect(
+ await mongoose.models.SkillUploadIntent.findOne({ fileId: input.file_id }).lean(),
+ ).toMatchObject({ storageKey: input.storageKey });
+ });
+
+ it('rejects moving a blob while reusing its file id', async () => {
+ const { skill } = await methods.createSkill(makeSkillInput());
+ const input = {
+ skillId: skill._id,
+ relativePath: 'references/stable.md',
+ file_id: 'stable-id',
+ filename: 'stable.md',
+ filepath: '/tmp/stable-id',
+ source: 'local',
+ mimeType: 'text/markdown',
+ bytes: 10,
+ author: owner._id,
+ };
+ await methods.upsertSkillFile(input);
+ await expect(
+ methods.upsertSkillFile({ ...input, filepath: '/tmp/other-object' }),
+ ).rejects.toMatchObject({
+ code: 'SKILL_FILE_CONFLICT',
+ });
+ expect(await methods.getSkillFileByPath(skill._id, input.relativePath)).toMatchObject({
+ file_id: 'stable-id',
+ filepath: '/tmp/stable-id',
+ });
+ expect(await mongoose.models.SkillUploadIntent.countDocuments({ fileId: 'stable-id' })).toBe(0);
+ });
+
it('creates only absent skill files and leaves the winner intact when creation races', async () => {
await SkillFile.init();
const { skill } = await methods.createSkill(makeSkillInput());
@@ -2457,11 +2624,24 @@ describe('deleteUserSkills', () => {
const deleted = await methods.deleteUserSkills(owner._id as mongoose.Types.ObjectId);
expect(deleted).toBe(1);
- expect(await Skill.countDocuments()).toBe(1);
+ // Account deletion queues storage-aware cleanup; it must not remove the
+ // Skill row before its blobs have been reconciled.
+ expect(await Skill.countDocuments()).toBe(2);
+ expect(await Skill.countDocuments({ deletingAt: { $exists: true } })).toBe(1);
+ expect(await Skill.countDocuments({ deletionNeedsJournal: true })).toBe(1);
+ expect(((await Skill.findById(mine._id).lean()) as { name: string } | null)?.name).toBe(
+ `__deleted__${mine._id}`,
+ );
expect(await Skill.countDocuments({ _id: sharedId })).toBe(1);
+ const { skill: replacement } = await methods.createSkill(makeSkillInput({ name: 'mine' }));
+ expect(replacement._id.toString()).not.toBe(mine._id.toString());
+ expect(await mongoose.models.SkillDeletion.countDocuments({ skillId: mine._id })).toBe(0);
+ await createSkillLifecycleMethods(mongoose).discoverUnjournaledDeletions();
+ expect(await mongoose.models.SkillDeletion.countDocuments({ skillId: mine._id })).toBe(1);
+ expect(await Skill.countDocuments({ deletionNeedsJournal: true })).toBe(0);
});
- it('prunes deleted sole-owned skill ids from agent allowlists', async () => {
+ it('defers allowlist pruning until storage-aware deletion finishes', async () => {
const { skill: mine } = await methods.createSkill(makeSkillInput({ name: 'mine' }));
await grantOwner(mine._id);
const Agent = mongoose.models.Agent;
@@ -2474,25 +2654,43 @@ describe('deleteUserSkills', () => {
skills?: string[];
skills_enabled?: boolean;
} | null;
- expect(agentAfter?.skills).toEqual([]);
- expect(agentAfter?.skills_enabled).toBe(false);
+ expect(agentAfter?.skills).toEqual([mine._id.toString()]);
+ expect(agentAfter?.skills_enabled).toBe(true);
});
- it('fails and keeps the retry anchor when agent allowlist cleanup fails', async () => {
+ it('retains the durable retry anchor when account-deletion allowlist cleanup fails', async () => {
const { skill: mine } = await methods.createSkill(makeSkillInput({ name: 'mine' }));
await grantOwner(mine._id);
const Agent = mongoose.models.Agent;
const agent = await Agent.create(makeAgentDoc([mine._id.toString(), 'keep-me']));
+ expect(await methods.deleteUserSkills(owner._id as mongoose.Types.ObjectId)).toBe(1);
+ expect(await Skill.findById(mine._id).lean()).toMatchObject({
+ deletionNeedsJournal: true,
+ });
+ // Account erasure removes principal ACLs before the background reconciler runs.
+ await AclEntry.deleteMany({ principalId: owner._id });
+ const lifecycle = createSkillLifecycleMethods(mongoose);
+ await lifecycle.discoverUnjournaledDeletions();
+
const failure = jest
.spyOn(Agent, 'updateMany')
.mockRejectedValueOnce(new Error('transient outage'));
- await expect(methods.deleteUserSkills(owner._id as mongoose.Types.ObjectId)).rejects.toThrow(
- 'Agent skill allowlists could not be cleaned up',
- );
- failure.mockRestore();
+ try {
+ expect(await methods.finalizeSkillDeletion(mine._id.toString())).toMatchObject({
+ cleanupComplete: false,
+ failedCleanupSteps: ['agent_allowlists'],
+ });
+ } finally {
+ failure.mockRestore();
+ }
+ expect(await Skill.findById(mine._id).lean()).not.toBeNull();
+ expect(await lifecycle.getSkillDeletion(mine._id.toString())).not.toBeNull();
- await methods.deleteUserSkills(owner._id as mongoose.Types.ObjectId);
+ expect(await methods.finalizeSkillDeletion(mine._id.toString())).toMatchObject({
+ cleanupComplete: true,
+ });
+ expect(await Skill.findById(mine._id).lean()).toBeNull();
const agentAfter = (await Agent.findById(agent._id).lean()) as { skills?: string[] } | null;
expect(agentAfter?.skills).toEqual(['keep-me']);
});
diff --git a/packages/data-schemas/src/methods/skill.ts b/packages/data-schemas/src/methods/skill.ts
index 2d47244d5f..0b371c2724 100644
--- a/packages/data-schemas/src/methods/skill.ts
+++ b/packages/data-schemas/src/methods/skill.ts
@@ -17,7 +17,9 @@ import type {
ISkillFileDocument,
ISkillSummary,
} from '~/types/skill';
+import type { ISkillUploadIntentDocument } from '~/types/skillDeletion';
import type { IAgent } from '~/types/agent';
+import { createSkillLifecycleMethods } from './skillLifecycle';
import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite';
import { isValidObjectIdString } from '~/utils/objectId';
import { stripYamlTrailingComment } from '~/utils/yaml';
@@ -754,6 +756,8 @@ export type UpsertSkillFileInput = {
expectedFileId?: string;
/** Insert only when the path is absent, even if another writer creates it first. */
createOnly?: boolean;
+ /** Keep the superseded blob until a multi-step writer commits or restores this file. */
+ cleanupHoldOwner?: string;
skillId: Types.ObjectId | string;
relativePath: string;
file_id: string;
@@ -1000,7 +1004,10 @@ export async function filterExistingSkillIds(
}
const Skill = mongoose.models.Skill as Model<ISkillDocument>;
const docs = await Skill.find(
- { _id: { $in: candidates.map((id) => new mongoose.Types.ObjectId(id)) } },
+ {
+ _id: { $in: candidates.map((id) => new mongoose.Types.ObjectId(id)) },
+ deletingAt: { $exists: false },
+ },
{ _id: 1 },
).lean<Array<{ _id: Types.ObjectId }>>();
const existing = new Set(docs.map((doc) => doc._id.toString()));
@@ -1053,6 +1060,8 @@ export function createSkillMethods(
update: UpdateSkillInput;
}) => Promise<UpdateSkillResult>;
deleteSkill: (id: string) => Promise<DeleteSkillResult>;
+ deleteSkillFileAfterBlob: (skillId: string, fileId: string) => Promise<boolean>;
+ finalizeSkillDeletion: (id: string) => Promise<DeleteSkillResult>;
deleteUserSkills: (userId: Types.ObjectId | string) => Promise<number>;
findSkillBySourceIdentity: (params: {
source: 'github' | 'notion';
@@ -1099,8 +1108,8 @@ export function createSkillMethods(
>,
): FilterQuery<ISkillDocument> {
const filter: FilterQuery<ISkillDocument> = params.manageTenantId
- ? { tenantId: params.manageTenantId }
- : { _id: { $in: params.accessibleIds } };
+ ? { tenantId: params.manageTenantId, deletingAt: { $exists: false } }
+ : { _id: { $in: params.accessibleIds }, deletingAt: { $exists: false } };
if (params.category && params.category.length > 0) {
filter.category = params.category;
}
@@ -1250,7 +1259,7 @@ export function createSkillMethods(
return null;
}
const Skill = mongoose.models.Skill as Model<ISkillDocument>;
- const doc = await Skill.findById(id).lean();
+ const doc = await Skill.findOne({ _id: id, deletingAt: { $exists: false } }).lean();
return (doc as unknown as (ISkill & { _id: Types.ObjectId }) | null) ?? null;
}
@@ -1288,7 +1297,11 @@ export function createSkillMethods(
the previous performance characteristics for callers that just
want "newest match". */
if (!preferUserInvocable && !preferModelInvocable) {
- const doc = await Skill.findOne({ name, _id: { $in: accessibleIds } })
+ const doc = await Skill.findOne({
+ name,
+ _id: { $in: accessibleIds },
+ deletingAt: { $exists: false },
+ })
.sort({ updatedAt: -1 })
.lean();
return backfillDerivedFromFrontmatter(
@@ -1298,7 +1311,11 @@ export function createSkillMethods(
/* Multi-doc path: fetch all matching docs (typically 1, rarely 2+
across same-name duplicates) and pick the first satisfying the
caller's preference; fall back to newest. */
- const docs = (await Skill.find({ name, _id: { $in: accessibleIds } })
+ const docs = (await Skill.find({
+ name,
+ _id: { $in: accessibleIds },
+ deletingAt: { $exists: false },
+ })
.sort({ updatedAt: -1 })
.lean()) as unknown as Array<ISkill & { _id: Types.ObjectId }>;
if (docs.length === 0) {
@@ -1327,6 +1344,7 @@ export function createSkillMethods(
name: params.name,
author: params.author,
tenantId: params.tenantId ?? null,
+ deletingAt: { $exists: false },
})
.sort({ updatedAt: -1 })
.lean();
@@ -1410,6 +1428,7 @@ export function createSkillMethods(
const baseFilter: FilterQuery<ISkillDocument> = {
_id: { $in: params.accessibleIds },
alwaysApply: true,
+ deletingAt: { $exists: false },
};
const cursor = decodeCursor(params.cursor);
@@ -1621,7 +1640,7 @@ export function createSkillMethods(
updateOps.$unset = unsetPayload;
}
const result = await Skill.findOneAndUpdate(
- { _id: new ObjectId(id), version: expectedVersion },
+ { _id: new ObjectId(id), version: expectedVersion, deletingAt: { $exists: false } },
updateOps,
{ new: true },
).lean();
@@ -1634,7 +1653,7 @@ export function createSkillMethods(
};
}
- const current = await Skill.findById(id).lean();
+ const current = await Skill.findOne({ _id: id, deletingAt: { $exists: false } }).lean();
if (!current) {
return { status: 'not_found' };
}
@@ -1712,67 +1731,106 @@ export function createSkillMethods(
failedCleanupSteps: [],
};
}
+ // Legacy callers may still reach this data-only method. It must never
+ // discard the last reference to an external blob. The storage-aware API
+ // reconciler owns provider deletion and finalization.
+ const lifecycle = createSkillLifecycleMethods(mongoose);
+ const journal = await lifecycle.beginSkillDeletion(id);
+ return {
+ deleted: false,
+ skillAbsent: !journal,
+ cleanupComplete: !journal,
+ failedCleanupSteps: journal ? ['skill_files'] : [],
+ };
+ }
+
+ /** Blob deletion is idempotent; the file row remains the retry manifest until it succeeds. */
+ async function deleteSkillFileAfterBlob(skillId: string, fileId: string): Promise<boolean> {
+ const SkillFile = mongoose.models.SkillFile as Model<ISkillFileDocument>;
+ const result = await SkillFile.deleteOne({
+ skillId: new ObjectId(skillId),
+ file_id: fileId,
+ });
+ return result.deletedCount === 1;
+ }
+
+ /** Finalize only after blob rows are drained; never remove late rows without deleting blobs. */
+ async function finalizeSkillDeletion(id: string): Promise<DeleteSkillResult> {
const Skill = mongoose.models.Skill as Model<ISkillDocument>;
const SkillFile = mongoose.models.SkillFile as Model<ISkillFileDocument>;
const objectId = new ObjectId(id);
- const res = await Skill.deleteOne({ _id: objectId });
+ const skill = await Skill.findById(objectId).select('_id deletingAt').lean();
+ if (skill && !skill.deletingAt) {
+ return {
+ deleted: false,
+ skillAbsent: false,
+ cleanupComplete: false,
+ failedCleanupSteps: ['skill_files'],
+ };
+ }
+ if (await SkillFile.exists({ skillId: objectId })) {
+ return {
+ deleted: false,
+ skillAbsent: false,
+ cleanupComplete: false,
+ failedCleanupSteps: ['skill_files'],
+ };
+ }
const failedCleanupSteps: DeleteSkillCleanupStep[] = [];
- const allowlistsRemoved = await removeSkillsFromAgentAllowlists([id]);
- if (!allowlistsRemoved) {
+ if (!(await removeSkillsFromAgentAllowlists([id]))) {
failedCleanupSteps.push('agent_allowlists');
}
-
- const [filesResult, permissionsResult] = await Promise.allSettled([
- SkillFile.deleteMany({ skillId: objectId }),
- deps.removeAllPermissions({ resourceType: ResourceType.SKILL, resourceId: id }),
- ]);
- if (filesResult.status === 'rejected') {
- failedCleanupSteps.push('skill_files');
- logger.error(`[deleteSkill] Error removing files for ${id}:`, filesResult.reason);
- }
- if (permissionsResult.status === 'rejected') {
+ try {
+ await deps.removeAllPermissions({ resourceType: ResourceType.SKILL, resourceId: id });
+ } catch {
failedCleanupSteps.push('permissions');
- logger.error(`[deleteSkill] Error removing permissions for ${id}:`, permissionsResult.reason);
+ logger.error(`[finalizeSkillDeletion] Error removing permissions for ${id}`);
}
-
+ if (failedCleanupSteps.length > 0) {
+ return {
+ deleted: false,
+ skillAbsent: false,
+ cleanupComplete: false,
+ failedCleanupSteps,
+ };
+ }
+ const result = await Skill.deleteOne({ _id: objectId, deletingAt: { $exists: true } });
return {
- deleted: Boolean(res.deletedCount),
+ deleted: result.deletedCount === 1,
skillAbsent: true,
- cleanupComplete: failedCleanupSteps.length === 0,
- failedCleanupSteps,
+ cleanupComplete: true,
+ failedCleanupSteps: [],
};
}
async function deleteUserSkills(userId: Types.ObjectId | string): Promise<number> {
const userObjectId = typeof userId === 'string' ? new ObjectId(userId) : userId;
- const Skill = mongoose.models.Skill as Model<ISkillDocument>;
const soleOwned = await deps.getSoleOwnedResourceIds(userObjectId, ResourceType.SKILL);
if (soleOwned.length === 0) {
return 0;
}
- const SkillFile = mongoose.models.SkillFile as Model<ISkillFileDocument>;
- await SkillFile.deleteMany({ skillId: { $in: soleOwned } });
- const res = await Skill.deleteMany({ _id: { $in: soleOwned } });
- if (!(await removeSkillsFromAgentAllowlists(soleOwned.map((rid) => rid.toString())))) {
- throw new Error('Agent skill allowlists could not be cleaned up');
- }
- /** Permission entries are the retry discovery key for sole-owned skills, so a failure keeps them. */
- const permissionResults = await Promise.allSettled(
- soleOwned.map((rid) =>
- deps.removeAllPermissions({
- resourceType: ResourceType.SKILL,
- resourceId: rid.toString(),
- }),
- ),
- );
- const failedPermission = permissionResults.find(
- (result): result is PromiseRejectedResult => result.status === 'rejected',
+ // Mark each Skill and release its public name in the same document write.
+ // The reconciler repairs the mark-to-journal gap and backfills file
+ // references outside this request.
+ const Skill = mongoose.models.Skill as Model<ISkillDocument>;
+ const deletingAt = new Date();
+ const result = await tenantSafeBulkWrite(
+ Skill,
+ soleOwned.map((id) => ({
+ updateOne: {
+ filter: { _id: id, deletingAt: { $exists: false } },
+ update: {
+ $set: {
+ name: `__deleted__${id.toString()}`,
+ deletingAt,
+ deletionNeedsJournal: true,
+ },
+ },
+ },
+ })),
+ { ordered: false },
);
- if (failedPermission) {
- logger.error('[deleteUserSkills] Error removing permissions:', failedPermission.reason);
- throw failedPermission.reason;
- }
- return res.deletedCount ?? 0;
+ return result.modifiedCount;
}
async function findSkillBySourceIdentity(params: {
@@ -1787,6 +1845,7 @@ export function createSkillMethods(
const doc = await Skill.findOne({
source: params.source,
'sourceMetadata.upstreamId': params.upstreamId,
+ deletingAt: { $exists: false },
...tenantFilter,
}).lean();
return (doc as unknown as (ISkill & { _id: Types.ObjectId }) | null) ?? null;
@@ -1800,6 +1859,7 @@ export function createSkillMethods(
const rows = await Skill.find({
source: params.source,
'sourceMetadata.sourceId': params.sourceId,
+ deletingAt: { $exists: false },
}).lean();
return rows as unknown as Array<ISkill & { _id: Types.ObjectId }>;
}
@@ -1851,6 +1911,16 @@ export function createSkillMethods(
async function upsertSkillFile(
row: UpsertSkillFileInput,
): Promise<ISkillFile & { _id: Types.ObjectId }> {
+ const Skill = mongoose.models.Skill as Model<ISkillDocument>;
+ const parent = await Skill.findOne({
+ _id: row.skillId,
+ deletingAt: { $exists: false },
+ })
+ .select('_id')
+ .lean();
+ if (!parent) {
+ throw Object.assign(new Error('Skill is being deleted'), { code: 'SKILL_DELETING' });
+ }
const issues = validateRelativePath(row.relativePath);
if (issues.length > 0) {
const error = new Error('Skill file validation failed');
@@ -1859,6 +1929,78 @@ export function createSkillMethods(
throw error;
}
const SkillFile = mongoose.models.SkillFile as Model<ISkillFileDocument>;
+ if (!row.createOnly) {
+ const previous = await SkillFile.findOne({
+ skillId: row.skillId,
+ relativePath: row.relativePath,
+ ...(row.expectedFileId != null ? { file_id: row.expectedFileId } : {}),
+ })
+ .select('-content')
+ .lean();
+ if (
+ previous &&
+ previous.file_id === row.file_id &&
+ (previous.filepath !== row.filepath ||
+ previous.source !== row.source ||
+ (previous.storageKey ?? null) !== (row.storageKey ?? null) ||
+ (previous.storageRegion ?? null) !== (row.storageRegion ?? null))
+ ) {
+ throw Object.assign(new Error('Skill file identity cannot change storage location'), {
+ code: 'SKILL_FILE_CONFLICT',
+ });
+ }
+ if (previous && previous.file_id !== row.file_id) {
+ // Queue the old blob before changing its only live file row. A failed
+ // CAS leaves the row in place, so the sweeper will retain its blob.
+ const SkillUploadIntent = mongoose.models
+ .SkillUploadIntent as Model<ISkillUploadIntentDocument>;
+ // A multi-step writer may roll this replacement back. Its hold and the
+ // sweeper's claim must contend on the same intent before the file CAS;
+ // an already-claimed blob is not a safe rollback candidate.
+ const intentFilter = row.cleanupHoldOwner
+ ? {
+ fileId: previous.file_id,
+ state: 'committed',
+ cleanedAt: { $exists: false },
+ claimOwner: { $exists: false },
+ $or: [
+ { cleanupHoldOwner: { $exists: false } },
+ { cleanupHoldOwner: row.cleanupHoldOwner },
+ ],
+ }
+ : { fileId: previous.file_id };
+ const intentUpdate = {
+ $setOnInsert: {
+ skillId: row.skillId,
+ fileId: previous.file_id,
+ source: previous.source,
+ storageKey: previous.storageKey ?? previous.filepath,
+ author: previous.author,
+ tenantId: previous.tenantId,
+ writerId: 'legacy-replacement',
+ state: 'committed',
+ },
+ $set: {
+ cleanupRequestedAt: new Date(),
+ ...(row.cleanupHoldOwner ? { cleanupHoldOwner: row.cleanupHoldOwner } : {}),
+ },
+ $unset: { cleanedAt: '', nextCheckAt: '' },
+ };
+ try {
+ await SkillUploadIntent.updateOne(intentFilter, intentUpdate, { upsert: true });
+ } catch (error) {
+ // Concurrent replacements can both attempt the legacy upsert.
+ // The winner has already published the same cleanup reference.
+ if (!(error instanceof Error && 'code' in error && error.code === 11000)) throw error;
+ const result = await SkillUploadIntent.updateOne(intentFilter, intentUpdate);
+ if (row.cleanupHoldOwner && result.matchedCount !== 1) {
+ throw Object.assign(new Error('Skill blob cleanup is already claimed'), {
+ code: 'SKILL_FILE_CONFLICT',
+ });
+ }
+ }
+ }
+ }
const fields = {
skillId: row.skillId,
relativePath: row.relativePath,
@@ -1927,6 +2069,16 @@ export function createSkillMethods(
relativePath: string,
expectedFileId?: string,
): Promise<{ deleted: boolean }> {
+ const Skill = mongoose.models.Skill as Model<ISkillDocument>;
+ const parent = await Skill.findOne({
+ _id: skillId,
+ deletingAt: { $exists: false },
+ })
+ .select('_id')
+ .lean();
+ if (!parent) {
+ throw Object.assign(new Error('Skill is being deleted'), { code: 'SKILL_DELETING' });
+ }
const SkillFile = mongoose.models.SkillFile as Model<ISkillFileDocument>;
const res = await SkillFile.deleteOne({
skillId,
@@ -2012,6 +2164,8 @@ export function createSkillMethods(
listAlwaysApplySkills,
updateSkill,
deleteSkill,
+ deleteSkillFileAfterBlob,
+ finalizeSkillDeletion,
deleteUserSkills,
findSkillBySourceIdentity,
listSkillsBySource,
diff --git a/packages/data-schemas/src/methods/skillDeletionAdapter.spec.ts b/packages/data-schemas/src/methods/skillDeletionAdapter.spec.ts
index 0e719d42a1..f86e618b36 100644
--- a/packages/data-schemas/src/methods/skillDeletionAdapter.spec.ts
+++ b/packages/data-schemas/src/methods/skillDeletionAdapter.spec.ts
@@ -5,10 +5,27 @@ it('projects deletion persistence records into plain string-ID API records', asy
const skillId = new Types.ObjectId();
const author = new Types.ObjectId();
const source = {
+ hasInFlightSkillUploads: jest.fn(async () => false),
+ listExistingSkillFileIds: jest.fn(async () => ['file-1']),
+ iterateSkillCleanupFiles: async function* () {
+ yield [{ skillId, author, file_id: 'file-1', filepath: '/uploads/file-1', source: 'local' }];
+ },
+ iterateSkillCleanupIntents: async function* () {
+ yield [
+ {
+ skillId,
+ author,
+ fileId: 'file-1',
+ source: 'local',
+ storageKey: '/uploads/file-1',
+ state: 'committed' as const,
+ },
+ ];
+ },
beginSkillDeletion: async () => ({ skillId, author, startedAt: new Date() }),
getSkillDeletion: async () => ({ skillId, author, startedAt: new Date() }),
listSkillDeletionTombstones: async () => [{ skillId, author, startedAt: new Date() }],
- listSkillUploadIntents: async () => [
+ listSkillUploadIntents: jest.fn(async () => [
{
skillId,
author,
@@ -18,7 +35,7 @@ it('projects deletion persistence records into plain string-ID API records', asy
writerId: 'writer',
state: 'committed' as const,
},
- ],
+ ]),
listAbandonedSkillUploadIntents: async () => [
{
skillId,
@@ -42,6 +59,18 @@ it('projects deletion persistence records into plain string-ID API records', asy
};
const adapter = createSkillDeletionPersistenceAdapter(source);
+ for await (const batch of adapter.iterateSkillCleanupFiles(skillId.toString(), 2)) {
+ expect(batch[0].author).toBe(author.toString());
+ }
+ for await (const batch of adapter.iterateSkillCleanupIntents(skillId.toString(), 2)) {
+ expect(batch[0]).toMatchObject({ skillId: skillId.toString(), author: author.toString() });
+ }
+ expect(await adapter.hasInFlightSkillUploads(skillId.toString())).toBe(false);
+ expect(await adapter.listExistingSkillFileIds(skillId.toString(), ['file-1'])).toEqual([
+ 'file-1',
+ ]);
+ expect(source.listExistingSkillFileIds).toHaveBeenCalledWith(skillId.toString(), ['file-1']);
+
expect(await adapter.beginSkillDeletion(skillId.toString())).toEqual({
skillId: skillId.toString(),
completedAt: undefined,
@@ -52,6 +81,9 @@ it('projects deletion persistence records into plain string-ID API records', asy
author: author.toString(),
fileId: 'file-1',
});
+ expect(source.listSkillUploadIntents).toHaveBeenCalledWith(skillId.toString(), {
+ cleanupOnly: true,
+ });
expect((await adapter.listAbandonedSkillUploadIntents(1))[0].author).toBe(author.toString());
expect((await adapter.listSkillFiles(skillId.toString()))[0]).toEqual({
file_id: 'file-1',
diff --git a/packages/data-schemas/src/methods/skillDeletionAdapter.ts b/packages/data-schemas/src/methods/skillDeletionAdapter.ts
index 913e074026..7146bb47ed 100644
--- a/packages/data-schemas/src/methods/skillDeletionAdapter.ts
+++ b/packages/data-schemas/src/methods/skillDeletionAdapter.ts
@@ -28,6 +28,37 @@ export interface SkillDeletionFileRecord {
}
type Source = {
+ iterateSkillCleanupFiles: (
+ id: string,
+ batchSize: number,
+ ) => AsyncIterable<
+ Array<
+ Pick<
+ ISkillFile,
+ 'file_id' | 'filepath' | 'source' | 'storageKey' | 'storageRegion' | 'author' | 'tenantId'
+ >
+ >
+ >;
+ iterateSkillCleanupIntents: (
+ id: string,
+ batchSize: number,
+ ) => AsyncIterable<
+ Array<
+ Pick<
+ ISkillUploadIntent,
+ | 'skillId'
+ | 'author'
+ | 'fileId'
+ | 'source'
+ | 'storageKey'
+ | 'tenantId'
+ | 'state'
+ | 'cleanedAt'
+ >
+ >
+ >;
+ hasInFlightSkillUploads: (id: string) => Promise<boolean>;
+ listExistingSkillFileIds: (id: string, fileIds: string[]) => Promise<string[]>;
beginSkillDeletion: (
id: string,
) => Promise<Pick<ISkillDeletion, 'skillId' | 'completedAt'> | null>;
@@ -37,6 +68,7 @@ type Source = {
) => Promise<Array<Pick<ISkillDeletion, 'skillId' | 'completedAt'>>>;
listSkillUploadIntents: (
id: string,
+ options?: { cleanupOnly?: boolean },
) => Promise<
Array<
Pick<
@@ -122,6 +154,16 @@ function file(
/** Project database records once, before they cross into the storage-independent API service. */
export function createSkillDeletionPersistenceAdapter(source: Source): {
+ iterateSkillCleanupFiles: (
+ id: string,
+ batchSize: number,
+ ) => AsyncIterable<SkillDeletionFileRecord[]>;
+ iterateSkillCleanupIntents: (
+ id: string,
+ batchSize: number,
+ ) => AsyncIterable<SkillUploadIntentRecord[]>;
+ hasInFlightSkillUploads: (id: string) => Promise<boolean>;
+ listExistingSkillFileIds: (id: string, fileIds: string[]) => Promise<string[]>;
beginSkillDeletion: (id: string) => Promise<SkillDeletionRecord | null>;
getSkillDeletion: (id: string) => Promise<SkillDeletionRecord | null>;
listSkillDeletionTombstones: (limit: number) => Promise<SkillDeletionRecord[]>;
@@ -130,6 +172,15 @@ export function createSkillDeletionPersistenceAdapter(source: Source): {
listSkillFiles: (id: string) => Promise<SkillDeletionFileRecord[]>;
} {
return {
+ iterateSkillCleanupFiles: async function* (id, batchSize) {
+ for await (const rows of source.iterateSkillCleanupFiles(id, batchSize)) yield rows.map(file);
+ },
+ iterateSkillCleanupIntents: async function* (id, batchSize) {
+ for await (const rows of source.iterateSkillCleanupIntents(id, batchSize))
+ yield rows.map(intent);
+ },
+ hasInFlightSkillUploads: (id) => source.hasInFlightSkillUploads(id),
+ listExistingSkillFileIds: (id, fileIds) => source.listExistingSkillFileIds(id, fileIds),
beginSkillDeletion: async (id: string): Promise<SkillDeletionRecord | null> => {
const row = await source.beginSkillDeletion(id);
return row ? journal(row) : null;
@@ -141,7 +192,7 @@ export function createSkillDeletionPersistenceAdapter(source: Source): {
listSkillDeletionTombstones: async (limit: number): Promise<SkillDeletionRecord[]> =>
(await source.listSkillDeletionTombstones(limit)).map(journal),
listSkillUploadIntents: async (id: string): Promise<SkillUploadIntentRecord[]> =>
- (await source.listSkillUploadIntents(id)).map(intent),
+ (await source.listSkillUploadIntents(id, { cleanupOnly: true })).map(intent),
listAbandonedSkillUploadIntents: async (limit: number): Promise<SkillUploadIntentRecord[]> =>
(await source.listAbandonedSkillUploadIntents(limit)).map(intent),
listSkillFiles: async (id: string): Promise<SkillDeletionFileRecord[]> =>
diff --git a/packages/data-schemas/src/methods/skillLifecycle.spec.ts b/packages/data-schemas/src/methods/skillLifecycle.spec.ts
index 5a4495a469..22963d0084 100644
--- a/packages/data-schemas/src/methods/skillLifecycle.spec.ts
+++ b/packages/data-schemas/src/methods/skillLifecycle.spec.ts
@@ -55,6 +55,98 @@ async function completeWithClaim(skillId: mongoose.Types.ObjectId) {
}
describe('Skill deletion journal', () => {
+ it('pages real file and intent cursors without skipping records changed during cleanup', async () => {
+ const { skill, author } = await makeSkill();
+ await mongoose.models.SkillFile.insertMany(
+ Array.from({ length: 205 }, (_, index) => ({
+ skillId: skill._id,
+ file_id: `paged-${index}`,
+ relativePath: `${index}.md`,
+ filename: `${index}.md`,
+ filepath: `/uploads/${author}/${index}`,
+ source: 'local',
+ mimeType: 'text/markdown',
+ bytes: 4,
+ content: 'must not be loaded during cleanup',
+ author,
+ tenantId: 'tenant-a',
+ })),
+ );
+ await lifecycle.beginSkillDeletion(skill.id);
+ const seen = new Set<string>();
+ for await (const files of lifecycle.iterateSkillCleanupFiles(skill.id, 17)) {
+ expect(files.length).toBeLessThanOrEqual(17);
+ for (const file of files) {
+ expect(file.content).toBeUndefined();
+ expect(seen.has(file.file_id)).toBe(false);
+ seen.add(file.file_id);
+ await mongoose.models.SkillFile.deleteOne({ file_id: file.file_id });
+ }
+ }
+ expect(seen.size).toBe(205);
+ expect(await lifecycle.hasInFlightSkillUploads(skill.id)).toBe(false);
+ const cleaned = new Set<string>();
+ for await (const intents of lifecycle.iterateSkillCleanupIntents(skill.id, 17)) {
+ expect(intents.length).toBeLessThanOrEqual(17);
+ expect(
+ await lifecycle.listExistingSkillFileIds(
+ skill.id,
+ intents.map((intent) => intent.fileId),
+ ),
+ ).toEqual([]);
+ for (const intent of intents) {
+ expect(cleaned.has(intent.fileId)).toBe(false);
+ cleaned.add(intent.fileId);
+ await lifecycle.markSkillUploadIntentCleaned(intent.fileId);
+ }
+ }
+ expect(cleaned.size).toBe(205);
+ for await (const _batch of lifecycle.iterateSkillCleanupIntents(skill.id, 17)) {
+ throw new Error('Cleaned committed history must not be yielded');
+ }
+ });
+
+ it.each(['break', 'throw'] as const)(
+ 'closes a Mongo cleanup cursor when its consumer exits with %s',
+ async (mode) => {
+ const { skill, author } = await makeSkill();
+ await mongoose.models.SkillUploadIntent.insertMany(
+ Array.from({ length: 7 }, (_, index) => ({
+ skillId: skill._id,
+ author,
+ fileId: `close-${index}`,
+ source: 'local',
+ storageKey: `/uploads/${index}`,
+ writerId: 'writer',
+ state: 'abandoned',
+ })),
+ );
+ const query = mongoose.models.SkillUploadIntent.find();
+ const find = jest.spyOn(mongoose.models.SkillUploadIntent, 'find').mockReturnValueOnce(query);
+ const cursor = jest.spyOn(query, 'cursor');
+ const pages = lifecycle.iterateSkillCleanupIntents(skill.id, 2);
+ expect(cursor).not.toHaveBeenCalled();
+ await pages[Symbol.asyncIterator]().next();
+ const close = jest.spyOn(cursor.mock.results[0].value, 'close');
+ try {
+ const consume = async () => {
+ for await (const batch of pages) {
+ expect(batch).toHaveLength(2);
+ if (mode === 'throw') throw new Error('consumer failure');
+ break;
+ }
+ };
+ if (mode === 'throw') await expect(consume()).rejects.toThrow('consumer failure');
+ else await consume();
+ expect(close).toHaveBeenCalledTimes(1);
+ } finally {
+ close.mockRestore();
+ cursor.mockRestore();
+ find.mockRestore();
+ }
+ },
+ );
+
it('indexes only marked Skills for startup and periodic deletion discovery', () => {
expect(mongoose.models.Skill.schema.indexes()).toContainEqual([
{ deletionNeedsJournal: 1, deletionJournalAttemptAt: 1, deletingAt: 1 },
@@ -62,6 +154,110 @@ describe('Skill deletion journal', () => {
]);
});
+ it('skips not-due completed history instead of scanning the idle tombstone corpus', async () => {
+ const model = mongoose.models.SkillDeletion;
+ await model.init();
+ const now = new Date();
+ await model.insertMany(
+ Array.from({ length: 1_000 }, () => ({
+ skillId: new mongoose.Types.ObjectId(),
+ author: new mongoose.Types.ObjectId(),
+ startedAt: now,
+ completedAt: now,
+ lastAttemptAt: now,
+ nextCheckAt: new Date(now.getTime() + 3_600_000),
+ })),
+ );
+ const find = jest.spyOn(model.collection, 'find');
+ expect(await lifecycle.listSkillDeletionTombstones()).toEqual([]);
+ const filter = find.mock.calls[0][0];
+ if (!filter) throw new Error('Expected the lifecycle query to reach MongoDB');
+ find.mockRestore();
+ const plan = await model.collection
+ .find(filter)
+ .sort({ lastAttemptAt: 1, startedAt: 1 })
+ .limit(100)
+ .explain('executionStats');
+ expect(plan.executionStats.nReturned).toBe(0);
+ expect(plan.executionStats.totalDocsExamined).toBeLessThan(50);
+ expect(plan.executionStats.totalKeysExamined).toBeLessThan(50);
+ });
+
+ it('excludes healthy committed uploads without scanning every live upload during idle cleanup', async () => {
+ const model = mongoose.models.SkillUploadIntent;
+ await model.init();
+ const skillId = new mongoose.Types.ObjectId();
+ const author = new mongoose.Types.ObjectId();
+ await model.insertMany(
+ Array.from({ length: 1_000 }, (_, index) => ({
+ skillId,
+ author,
+ fileId: `healthy-${index}`,
+ writerId: 'finished-writer',
+ state: 'committed',
+ source: 'local',
+ storageKey: `/uploads/${author}/healthy-${index}`,
+ })),
+ );
+ const find = jest.spyOn(model.collection, 'find');
+ expect(await lifecycle.listAbandonedSkillUploadIntents()).toEqual([]);
+ const filter = find.mock.calls[0][0];
+ if (!filter) throw new Error('Expected the lifecycle query to reach MongoDB');
+ find.mockRestore();
+ const plan = await model.collection
+ .find(filter)
+ .sort({ lastCleanupAttemptAt: 1, createdAt: 1 })
+ .limit(100)
+ .explain('executionStats');
+ expect(plan.executionStats.nReturned).toBe(0);
+ expect(plan.executionStats.totalDocsExamined).toBeLessThan(50);
+ expect(plan.executionStats.totalKeysExamined).toBeLessThan(50);
+ });
+
+ it('keeps inspection history while cleanup excludes only already-cleaned committed uploads', async () => {
+ const model = mongoose.models.SkillUploadIntent;
+ await model.init();
+ const skillId = new mongoose.Types.ObjectId();
+ const author = new mongoose.Types.ObjectId();
+ const base = {
+ skillId,
+ author,
+ writerId: 'writer',
+ source: 'local',
+ storageKey: '/uploads/fixture',
+ };
+ await model.insertMany([
+ ...Array.from({ length: 1_000 }, (_, index) => ({
+ ...base,
+ fileId: `history-${index}`,
+ state: 'committed',
+ cleanedAt: new Date(),
+ })),
+ { ...base, fileId: 'uploading', state: 'uploading' },
+ { ...base, fileId: 'committing', state: 'committing' },
+ { ...base, fileId: 'uncleaned-committed', state: 'committed' },
+ { ...base, fileId: 'abandoned', state: 'abandoned' },
+ { ...base, fileId: 'cleaned-abandoned', state: 'abandoned', cleanedAt: new Date() },
+ ]);
+ expect(await lifecycle.listSkillUploadIntents(skillId)).toHaveLength(1_005);
+ const find = jest.spyOn(model.collection, 'find');
+ const cleanup = await lifecycle.listSkillUploadIntents(skillId, { cleanupOnly: true });
+ const filter = find.mock.calls[0][0];
+ find.mockRestore();
+ expect(cleanup.map((intent) => intent.fileId).sort()).toEqual([
+ 'abandoned',
+ 'cleaned-abandoned',
+ 'committing',
+ 'uncleaned-committed',
+ 'uploading',
+ ]);
+ if (!filter) throw new Error('Expected the cleanup query to reach MongoDB');
+ const plan = await model.collection.find(filter).explain('executionStats');
+ expect(plan.executionStats.nReturned).toBe(5);
+ expect(plan.executionStats.totalDocsExamined).toBeLessThan(50);
+ expect(plan.executionStats.totalKeysExamined).toBeLessThan(50);
+ });
+
it('marks a skill before recording its retryable tombstone', async () => {
const { skill } = await makeSkill();
const result = await lifecycle.beginSkillDeletion(skill.id);
@@ -677,6 +873,58 @@ describe('Skill upload intents', () => {
).toBeInstanceOf(Date);
});
+ it('retains an existing cleanup lease after cleaning a committed intent without making it claimable again', async () => {
+ const { skill, author } = await makeSkill();
+ const fileId = 'cleaned-committed';
+ await mongoose.models.SkillUploadIntent.create({
+ skillId: skill._id,
+ fileId,
+ source: 'local',
+ storageKey: `/uploads/${author}/${fileId}`,
+ author,
+ writerId: 'finished-writer',
+ state: 'committed',
+ cleanupRequestedAt: new Date(),
+ });
+ expect(await lifecycle.claimAbandonedSkillUploadIntent(fileId, 'worker-a', 60_000)).toBe(true);
+ await lifecycle.markSkillUploadIntentCleaned(fileId);
+ expect(await lifecycle.refreshAbandonedSkillUploadIntentClaim(fileId, 'worker-a', 60_000)).toBe(
+ true,
+ );
+ expect(await lifecycle.refreshAbandonedSkillUploadIntentClaim(fileId, 'worker-b', 60_000)).toBe(
+ false,
+ );
+ expect(
+ await lifecycle.scheduleAbandonedSkillUploadIntentRecheck(fileId, 'worker-b', 60_000),
+ ).toBe(false);
+ expect(
+ await lifecycle.scheduleAbandonedSkillUploadIntentRecheck(fileId, 'worker-a', 60_000),
+ ).toBe(true);
+ await lifecycle.noteAbandonedSkillUploadIntentAttempt(fileId);
+ const completed = await lifecycle.getSkillUploadIntent(fileId);
+ expect(completed?.cleanedAt).toBeInstanceOf(Date);
+ expect(completed?.lastCleanupAttemptAt).toBeInstanceOf(Date);
+
+ await mongoose.models.SkillUploadIntent.updateOne(
+ { fileId },
+ { $set: { claimExpiresAt: new Date(Date.now() - 1) } },
+ );
+ expect(await lifecycle.refreshAbandonedSkillUploadIntentClaim(fileId, 'worker-a', 60_000)).toBe(
+ false,
+ );
+ expect(
+ await lifecycle.scheduleAbandonedSkillUploadIntentRecheck(fileId, 'worker-a', 60_000),
+ ).toBe(false);
+ await lifecycle.releaseAbandonedSkillUploadIntentClaim(fileId, 'worker-a');
+ await mongoose.models.SkillUploadIntent.updateOne(
+ { fileId },
+ { $set: { nextCheckAt: new Date(Date.now() - 1) } },
+ );
+ expect(await lifecycle.listAbandonedSkillUploadIntents()).toEqual([]);
+ expect(await lifecycle.claimAbandonedSkillUploadIntent(fileId, 'worker-b', 60_000)).toBe(false);
+ expect((await lifecycle.getSkillUploadIntent(fileId))?.claimOwner).toBeUndefined();
+ });
+
it('keeps cleaned abandoned intents as deferred late-upload sentinels', async () => {
const { skill, author } = await makeSkill();
await lifecycle.reserveSkillUploadIntent({
@@ -742,4 +990,131 @@ describe('Skill upload intents', () => {
);
expect(await lifecycle.listAbandonedSkillUploadIntents()).toHaveLength(1);
});
+
+ it('keeps a held committed intent outside every cleanup sweep path until its owner releases it', async () => {
+ const { skill, author } = await makeSkill();
+ await mongoose.models.SkillUploadIntent.create({
+ skillId: skill._id,
+ fileId: 'held-file',
+ source: 'local',
+ storageKey: `/uploads/${author}/held-file`,
+ author,
+ writerId: 'old-writer',
+ state: 'committed',
+ cleanupRequestedAt: new Date(),
+ cleanupHoldOwner: 'replacement-writer',
+ claimOwner: 'sweeper',
+ claimExpiresAt: new Date(Date.now() + 60_000),
+ });
+
+ expect(await lifecycle.listAbandonedSkillUploadIntents()).toEqual([]);
+ expect(await lifecycle.claimAbandonedSkillUploadIntent('held-file', 'sweeper', 60_000)).toBe(
+ false,
+ );
+ expect(
+ await lifecycle.refreshAbandonedSkillUploadIntentClaim('held-file', 'sweeper', 60_000),
+ ).toBe(false);
+ expect(
+ await lifecycle.scheduleAbandonedSkillUploadIntentRecheck('held-file', 'sweeper', 60_000),
+ ).toBe(false);
+ expect(await lifecycle.finishSkillUploadCleanupHold('held-file', 'wrong-writer', true)).toBe(
+ false,
+ );
+ expect(
+ await lifecycle.finishSkillUploadCleanupHold('held-file', 'replacement-writer', false),
+ ).toBe(true);
+ expect((await lifecycle.getSkillUploadIntent('held-file'))?.cleanupRequestedAt).toBeUndefined();
+ expect((await lifecycle.getSkillUploadIntent('held-file'))?.cleanupHoldOwner).toBeUndefined();
+ expect(await lifecycle.listAbandonedSkillUploadIntents()).toEqual([]);
+ expect(
+ await lifecycle.finishSkillUploadCleanupHold('held-file', 'replacement-writer', true),
+ ).toBe(false);
+ });
+
+ it('releases a held original for cleanup when the replacement removed its file row', async () => {
+ const { skill, author } = await makeSkill();
+ await mongoose.models.SkillUploadIntent.create({
+ skillId: skill._id,
+ fileId: 'replaced-file',
+ source: 'local',
+ storageKey: `/uploads/${author}/replaced-file`,
+ author,
+ writerId: 'old-writer',
+ state: 'committed',
+ cleanupHoldOwner: 'replacement-writer',
+ nextCheckAt: new Date(Date.now() + 3_600_000),
+ });
+
+ expect(
+ await lifecycle.finishSkillUploadCleanupHold('replaced-file', 'replacement-writer', true),
+ ).toBe(true);
+ const intent = await lifecycle.getSkillUploadIntent('replaced-file');
+ expect(intent?.cleanupHoldOwner).toBeUndefined();
+ expect(intent?.cleanupRequestedAt).toBeInstanceOf(Date);
+ expect(intent?.nextCheckAt).toBeUndefined();
+ expect(
+ (await lifecycle.listAbandonedSkillUploadIntents()).map((item) => item.fileId),
+ ).toContain('replaced-file');
+ });
+
+ it('recovers a stopped cleanup hold according to the original file row', async () => {
+ const { skill, author } = await makeSkill();
+ await mongoose.models.SkillUploadIntent.create([
+ {
+ skillId: skill._id,
+ fileId: 'still-present',
+ source: 'local',
+ storageKey: `/uploads/${author}/still-present`,
+ author,
+ writerId: 'old-writer',
+ state: 'committed',
+ cleanupHoldOwner: 'stopped-writer',
+ },
+ {
+ skillId: skill._id,
+ fileId: 'removed',
+ source: 'local',
+ storageKey: `/uploads/${author}/removed`,
+ author,
+ writerId: 'old-writer',
+ state: 'committed',
+ cleanupHoldOwner: 'stopped-writer',
+ },
+ ]);
+ await mongoose.models.SkillFile.create({
+ skillId: skill._id,
+ file_id: 'still-present',
+ relativePath: 'references/still-present.md',
+ filename: 'still-present.md',
+ filepath: `/uploads/${author}/still-present`,
+ source: 'local',
+ mimeType: 'text/markdown',
+ bytes: 4,
+ author,
+ });
+
+ expect(
+ await lifecycle.recoverStoppedSkillUploadCleanupHold('missing', 'stopped-writer'),
+ ).toEqual({
+ status: 'missing',
+ });
+ expect(
+ await lifecycle.recoverStoppedSkillUploadCleanupHold('still-present', 'wrong-writer'),
+ ).toEqual({ status: 'changed' });
+ expect(
+ await lifecycle.recoverStoppedSkillUploadCleanupHold('still-present', 'stopped-writer'),
+ ).toEqual({ status: 'recovered', cleanup: false });
+ expect(
+ (await lifecycle.getSkillUploadIntent('still-present'))?.cleanupRequestedAt,
+ ).toBeUndefined();
+ expect(
+ await lifecycle.recoverStoppedSkillUploadCleanupHold('removed', 'stopped-writer'),
+ ).toEqual({
+ status: 'recovered',
+ cleanup: true,
+ });
+ expect((await lifecycle.listAbandonedSkillUploadIntents()).map((item) => item.fileId)).toEqual([
+ 'removed',
+ ]);
+ });
});
diff --git a/packages/data-schemas/src/methods/skillLifecycle.ts b/packages/data-schemas/src/methods/skillLifecycle.ts
index 803d2cbaf8..623fb1f22b 100644
--- a/packages/data-schemas/src/methods/skillLifecycle.ts
+++ b/packages/data-schemas/src/methods/skillLifecycle.ts
@@ -5,7 +5,7 @@ import type {
ISkillUploadIntent,
ISkillUploadIntentDocument,
} from '~/types/skillDeletion';
-import type { ISkillFileDocument } from '~/types/skill';
+import type { ISkillFile, ISkillFileDocument } from '~/types/skill';
import type { ISkillDocument } from '~/types/skill';
import { tenantSafeBulkWrite } from '~/utils/tenantBulkWrite';
import { runAsSystem } from '~/config/tenantContext';
@@ -13,6 +13,9 @@ import logger from '~/config/winston';
type SkillRow = ISkillDocument & { _id: Types.ObjectId };
const LEGACY_BACKFILL_BATCH_SIZE = 100;
+const pendingCleanupFilter = {
+ $or: [{ state: { $ne: 'committed' } }, { state: 'committed', cleanedAt: { $exists: false } }],
+};
type IntentInput = {
skillId: string;
author: string;
@@ -23,10 +26,35 @@ type IntentInput = {
writerId: string;
};
+async function* cleanupCursorBatches<Row>(
+ cursor: { next: () => Promise<Row | null>; close: () => Promise<void> },
+ batchSize: number,
+): AsyncGenerator<Row[]> {
+ try {
+ if (!Number.isInteger(batchSize) || batchSize < 1)
+ throw new Error('Invalid cleanup batch size');
+ let batch: Row[] = [];
+ for (let row = await cursor.next(); row != null; row = await cursor.next()) {
+ batch.push(row);
+ if (batch.length === batchSize) {
+ yield batch;
+ batch = [];
+ }
+ }
+ if (batch.length > 0) yield batch;
+ } finally {
+ await cursor.close();
+ }
+}
+
export type StoppedSkillUploadRecovery =
| { status: 'missing' | 'changed' }
| { status: 'recovered'; state: 'committed' | 'abandoned' };
+export type StoppedSkillUploadCleanupHoldRecovery =
+ | { status: 'missing' | 'changed' }
+ | { status: 'recovered'; cleanup: boolean };
+
export interface SkillLifecycleMethods {
beginSkillDeletion: (id: string) => Promise<ISkillDeletion | null>;
getSkillDeletion: (id: string) => Promise<ISkillDeletion | null>;
@@ -59,12 +87,31 @@ export interface SkillLifecycleMethods {
from: ISkillUploadIntent['state'],
to: ISkillUploadIntent['state'],
) => Promise<boolean>;
- listSkillUploadIntents: (skillId: string | Types.ObjectId) => Promise<ISkillUploadIntent[]>;
+ listSkillUploadIntents: (
+ skillId: string | Types.ObjectId,
+ options?: { cleanupOnly?: boolean },
+ ) => Promise<ISkillUploadIntent[]>;
+ iterateSkillCleanupFiles: (skillId: string, batchSize: number) => AsyncIterable<ISkillFile[]>;
+ iterateSkillCleanupIntents: (
+ skillId: string,
+ batchSize: number,
+ ) => AsyncIterable<ISkillUploadIntent[]>;
+ hasInFlightSkillUploads: (skillId: string) => Promise<boolean>;
+ listExistingSkillFileIds: (skillId: string, fileIds: string[]) => Promise<string[]>;
getSkillUploadIntent: (fileId: string) => Promise<ISkillUploadIntent | null>;
recoverStoppedSkillUploadIntent: (
fileId: string,
expectedWriterId: string,
) => Promise<StoppedSkillUploadRecovery>;
+ finishSkillUploadCleanupHold: (
+ fileId: string,
+ owner: string,
+ cleanup: boolean,
+ ) => Promise<boolean>;
+ recoverStoppedSkillUploadCleanupHold: (
+ fileId: string,
+ expectedOwner: string,
+ ) => Promise<StoppedSkillUploadCleanupHoldRecovery>;
listAbandonedSkillUploadIntents: (limit?: number) => Promise<ISkillUploadIntent[]>;
claimAbandonedSkillUploadIntent: (
fileId: string,
@@ -135,7 +182,7 @@ export function createSkillLifecycleMethods(
// The marker also covers crashes after the journal insert but before
// every legacy file has a durable blob reference.
let writes: AnyBulkWriteOperation[] = [];
- for await (const file of skillFileModel().find({ skillId }).cursor()) {
+ for await (const file of skillFileModel().find({ skillId }).select('-content').cursor()) {
writes.push({
updateOne: {
filter: {
@@ -355,14 +402,61 @@ export function createSkillLifecycleMethods(
async function listSkillUploadIntents(
skillId: string | Types.ObjectId,
+ options: { cleanupOnly?: boolean } = {},
): Promise<ISkillUploadIntent[]> {
- return (await intentModel().find({ skillId }).lean()) as ISkillUploadIntent[];
+ return (await intentModel()
+ .find({
+ skillId,
+ ...(options.cleanupOnly ? pendingCleanupFilter : {}),
+ })
+ .lean()) as ISkillUploadIntent[];
}
async function getSkillUploadIntent(fileId: string): Promise<ISkillUploadIntent | null> {
return (await intentModel().findOne({ fileId }).lean()) as ISkillUploadIntent | null;
}
+ async function* iterateSkillCleanupFiles(
+ skillId: string,
+ batchSize: number,
+ ): AsyncGenerator<ISkillFile[]> {
+ const cursor = skillFileModel()
+ .find({ skillId })
+ .select('-content')
+ .lean()
+ .cursor({ batchSize });
+ yield* cleanupCursorBatches<ISkillFile>(cursor, batchSize);
+ }
+
+ async function* iterateSkillCleanupIntents(
+ skillId: string,
+ batchSize: number,
+ ): AsyncGenerator<ISkillUploadIntent[]> {
+ const cursor = intentModel()
+ .find({
+ skillId,
+ ...pendingCleanupFilter,
+ })
+ .lean()
+ .cursor({ batchSize });
+ yield* cleanupCursorBatches<ISkillUploadIntent>(cursor, batchSize);
+ }
+
+ async function hasInFlightSkillUploads(skillId: string): Promise<boolean> {
+ return Boolean(
+ await intentModel().exists({ skillId, state: { $in: ['uploading', 'committing'] } }),
+ );
+ }
+
+ async function listExistingSkillFileIds(skillId: string, fileIds: string[]): Promise<string[]> {
+ if (fileIds.length === 0) return [];
+ const rows = await skillFileModel()
+ .find({ skillId, file_id: { $in: fileIds } })
+ .select('file_id')
+ .lean();
+ return rows.map((row) => row.file_id);
+ }
+
/** Operator-only recovery: call only after independently confirming the writer has stopped. */
async function recoverStoppedSkillUploadIntent(
fileId: string,
@@ -385,11 +479,52 @@ export function createSkillLifecycleMethods(
return result.modifiedCount === 1 ? { status: 'recovered', state } : { status: 'changed' };
}
+ async function finishSkillUploadCleanupHold(
+ fileId: string,
+ owner: string,
+ cleanup: boolean,
+ ): Promise<boolean> {
+ const result = await intentModel().updateOne(
+ { fileId, state: 'committed', cleanupHoldOwner: owner },
+ cleanup
+ ? {
+ $set: { cleanupRequestedAt: new Date() },
+ $unset: { cleanupHoldOwner: '', nextCheckAt: '' },
+ }
+ : { $unset: { cleanupHoldOwner: '', cleanupRequestedAt: '' } },
+ );
+ return result.matchedCount === 1;
+ }
+
+ /** Operator-only recovery after confirming the replacement writer has stopped. */
+ async function recoverStoppedSkillUploadCleanupHold(
+ fileId: string,
+ expectedOwner: string,
+ ): Promise<StoppedSkillUploadCleanupHoldRecovery> {
+ const intent = await getSkillUploadIntent(fileId);
+ if (!intent) return { status: 'missing' };
+ if (intent.state !== 'committed' || intent.cleanupHoldOwner !== expectedOwner) {
+ return { status: 'changed' };
+ }
+ const rowExists = await skillFileExistsForIntent(intent.skillId.toString(), fileId);
+ const cleanup = !rowExists;
+ const released = await finishSkillUploadCleanupHold(fileId, expectedOwner, cleanup);
+ return released ? { status: 'recovered', cleanup } : { status: 'changed' };
+ }
+
async function listAbandonedSkillUploadIntents(limit = 100): Promise<ISkillUploadIntent[]> {
const now = new Date();
return (await intentModel()
.find({
- state: 'abandoned',
+ cleanupHoldOwner: { $exists: false },
+ $or: [
+ { state: 'abandoned' },
+ {
+ state: 'committed',
+ cleanupRequestedAt: { $exists: true },
+ cleanedAt: { $exists: false },
+ },
+ ],
$and: [
{ $or: [{ claimExpiresAt: { $exists: false } }, { claimExpiresAt: { $lte: now } }] },
{ $or: [{ nextCheckAt: { $exists: false } }, { nextCheckAt: { $lte: now } }] },
@@ -409,7 +544,15 @@ export function createSkillLifecycleMethods(
const result = await intentModel().updateOne(
{
fileId,
- state: 'abandoned',
+ cleanupHoldOwner: { $exists: false },
+ $or: [
+ { state: 'abandoned' },
+ {
+ state: 'committed',
+ cleanupRequestedAt: { $exists: true },
+ cleanedAt: { $exists: false },
+ },
+ ],
$and: [
{
$or: [
@@ -431,8 +574,21 @@ export function createSkillLifecycleMethods(
owner: string,
leaseMs: number,
): Promise<boolean> {
+ // Finishing cleanup changes eligibility for a new claim, not ownership of this lease.
const result = await intentModel().updateOne(
- { fileId, state: 'abandoned', claimOwner: owner, claimExpiresAt: { $gt: new Date() } },
+ {
+ fileId,
+ cleanupHoldOwner: { $exists: false },
+ claimOwner: owner,
+ claimExpiresAt: { $gt: new Date() },
+ $or: [
+ { state: 'abandoned' },
+ {
+ state: 'committed',
+ cleanupRequestedAt: { $exists: true },
+ },
+ ],
+ },
{ $set: { claimExpiresAt: new Date(Date.now() + leaseMs) } },
);
return result.matchedCount === 1;
@@ -482,9 +638,16 @@ export function createSkillLifecycleMethods(
const result = await intentModel().updateOne(
{
fileId,
- state: 'abandoned',
+ cleanupHoldOwner: { $exists: false },
claimOwner: owner,
claimExpiresAt: { $gt: now },
+ $or: [
+ { state: 'abandoned' },
+ {
+ state: 'committed',
+ cleanupRequestedAt: { $exists: true },
+ },
+ ],
},
{ $set: { nextCheckAt: new Date(now.getTime() + delayMs) } },
);
@@ -507,8 +670,14 @@ export function createSkillLifecycleMethods(
reserveSkillUploadIntent,
transitionSkillUploadIntent,
listSkillUploadIntents,
+ iterateSkillCleanupFiles,
+ iterateSkillCleanupIntents,
+ hasInFlightSkillUploads,
+ listExistingSkillFileIds,
getSkillUploadIntent,
recoverStoppedSkillUploadIntent,
+ finishSkillUploadCleanupHold,
+ recoverStoppedSkillUploadCleanupHold,
listAbandonedSkillUploadIntents,
claimAbandonedSkillUploadIntent,
refreshAbandonedSkillUploadIntentClaim,
diff --git a/packages/data-schemas/src/schema/skillDeletion.ts b/packages/data-schemas/src/schema/skillDeletion.ts
index a7e081f738..ddf89aa48b 100644
--- a/packages/data-schemas/src/schema/skillDeletion.ts
+++ b/packages/data-schemas/src/schema/skillDeletion.ts
@@ -19,5 +19,7 @@ const skillDeletionSchema: Schema<ISkillDeletionDocument> = new Schema<ISkillDel
// The sweep rotates attempted rows and sorts by these fields before its batch limit.
skillDeletionSchema.index({ lastAttemptAt: 1, startedAt: 1, claimExpiresAt: 1 });
skillDeletionSchema.index({ completedAt: 1, nextCheckAt: 1 });
+// Let idle sweeps skip completed history whose next audit is not due.
+skillDeletionSchema.index({ nextCheckAt: 1 });
export default skillDeletionSchema;
diff --git a/packages/data-schemas/src/schema/skillUploadIntent.ts b/packages/data-schemas/src/schema/skillUploadIntent.ts
index a96e8243ad..37ce30e1e0 100644
--- a/packages/data-schemas/src/schema/skillUploadIntent.ts
+++ b/packages/data-schemas/src/schema/skillUploadIntent.ts
@@ -16,6 +16,8 @@ const skillUploadIntentSchema: Schema<ISkillUploadIntentDocument> =
enum: ['uploading', 'committing', 'committed', 'abandoned'],
required: true,
},
+ cleanupHoldOwner: String,
+ cleanupRequestedAt: Date,
cleanedAt: Date,
nextCheckAt: Date,
lastCleanupAttemptAt: Date,
@@ -26,7 +28,17 @@ const skillUploadIntentSchema: Schema<ISkillUploadIntentDocument> =
);
skillUploadIntentSchema.index({ skillId: 1, state: 1 });
+skillUploadIntentSchema.index({ skillId: 1, state: 1, cleanedAt: 1 });
skillUploadIntentSchema.index({ state: 1, lastCleanupAttemptAt: 1, createdAt: 1 });
+// The committed cleanup branch must not scan healthy live uploads just to preserve ordering.
+skillUploadIntentSchema.index(
+ { state: 1, lastCleanupAttemptAt: 1, createdAt: 1 },
+ {
+ name: 'committed_cleanup_candidates',
+ partialFilterExpression: { state: 'committed', cleanupRequestedAt: { $exists: true } },
+ },
+);
skillUploadIntentSchema.index({ state: 1, cleanedAt: 1, nextCheckAt: 1 });
+skillUploadIntentSchema.index({ cleanupRequestedAt: 1, state: 1, nextCheckAt: 1 });
export default skillUploadIntentSchema;
diff --git a/packages/data-schemas/src/types/skillDeletion.ts b/packages/data-schemas/src/types/skillDeletion.ts
index 7507ccb9ab..24c9465af6 100644
--- a/packages/data-schemas/src/types/skillDeletion.ts
+++ b/packages/data-schemas/src/types/skillDeletion.ts
@@ -27,6 +27,8 @@ export interface ISkillUploadIntent {
tenantId?: string;
writerId: string;
state: 'uploading' | 'committing' | 'committed' | 'abandoned';
+ cleanupHoldOwner?: string;
+ cleanupRequestedAt?: Date;
cleanedAt?: Date;
nextCheckAt?: Date;
lastCleanupAttemptAt?: Date;