Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧹 feat: Add Bounded Whole-Skill Cleanup Engine (#17042)

Dustin Healyc1815782738d
diff --git a/packages/api/src/skills/cleanup.spec.ts b/packages/api/src/skills/cleanup.spec.ts
index b974a241d7..f43d021c12 100644
--- a/packages/api/src/skills/cleanup.spec.ts
+++ b/packages/api/src/skills/cleanup.spec.ts
@@ -30,6 +30,34 @@ describe('mergeDeleteSkillResults', () => {
failedCleanupSteps: ['permissions'],
});
});
+
+ it('does not lose a pending upload intent when a retry cannot confirm Skill absence', () => {
+ expect(
+ mergeDeleteSkillResults(incomplete('upload_intents'), {
+ deleted: false,
+ skillAbsent: false,
+ cleanupComplete: false,
+ failedCleanupSteps: [],
+ }),
+ ).toEqual({
+ deleted: true,
+ skillAbsent: true,
+ cleanupComplete: false,
+ failedCleanupSteps: ['upload_intents'],
+ });
+ });
+
+ it('does not treat a late upload intent as settled by an earlier clean pass', () => {
+ expect(
+ mergeDeleteSkillResults(
+ { deleted: true, skillAbsent: true, cleanupComplete: true, failedCleanupSteps: [] },
+ incomplete('upload_intents'),
+ ),
+ ).toMatchObject({
+ cleanupComplete: false,
+ failedCleanupSteps: ['upload_intents'],
+ });
+ });
});
describe('deleteSkillWithRetry', () => {
diff --git a/packages/api/src/skills/cleanup.ts b/packages/api/src/skills/cleanup.ts
index d0b6a35d33..77c8241da4 100644
--- a/packages/api/src/skills/cleanup.ts
+++ b/packages/api/src/skills/cleanup.ts
@@ -1,21 +1,33 @@
import type { DeleteSkillCleanupStep, DeleteSkillResult } from '@librechat/data-schemas';
-const CLEANUP_STEPS: DeleteSkillCleanupStep[] = ['agent_allowlists', 'skill_files', 'permissions'];
+const CLEANUP_STEPS: DeleteSkillCleanupStep[] = [
+ 'agent_allowlists',
+ 'skill_files',
+ 'upload_intents',
+ 'permissions',
+];
/**
- * Cleanup is monotonic across idempotent deletion attempts. Once any attempt
- * confirms a step succeeded, a later transient failure cannot make that work
- * incomplete again. Only steps that failed in every confirming attempt remain
- * unsettled.
+ * Most cleanup steps are monotonic across idempotent attempts: one confirmed
+ * success settles a transient failure. Upload intents are different because a
+ * late writer can create one after an earlier pass, so the latest confirming
+ * result decides whether that step is settled.
*/
export function mergeDeleteSkillResults(
current: DeleteSkillResult,
next: DeleteSkillResult,
): DeleteSkillResult {
const confirmingAttempts = [current, next].filter((result) => result.skillAbsent);
- const failedCleanupSteps = CLEANUP_STEPS.filter((step) =>
- confirmingAttempts.every((result) => result.failedCleanupSteps.includes(step)),
- );
+ const failedCleanupSteps = CLEANUP_STEPS.filter((step) => {
+ if (step === 'upload_intents') {
+ // A late writer can introduce a new intent after an earlier clean pass.
+ // Only a newer confirming pass may settle this non-monotonic step.
+ return next.skillAbsent
+ ? next.failedCleanupSteps.includes(step)
+ : current.failedCleanupSteps.includes(step);
+ }
+ return confirmingAttempts.every((result) => result.failedCleanupSteps.includes(step));
+ });
const skillAbsent = confirmingAttempts.length > 0;
return {
diff --git a/packages/api/src/skills/deletion.spec.ts b/packages/api/src/skills/deletion.spec.ts
new file mode 100644
index 0000000000..f607de0b72
--- /dev/null
+++ b/packages/api/src/skills/deletion.spec.ts
@@ -0,0 +1,300 @@
+import type { SkillDeletionRecord, SkillUploadIntentRecord } from '@librechat/data-schemas';
+import type { SkillDeletionDeps } from './deletion';
+import { runSkillDeletion, SkillDeletionClaimLostError } from './deletion';
+
+const skillId = '507f1f77bcf86cd799439011';
+const author = '507f1f77bcf86cd799439012';
+const journal = {
+ skillId,
+} satisfies SkillDeletionRecord;
+
+function harness() {
+ const files = [
+ {
+ file_id: 'file-1',
+ filepath: '/uploads/owner/file-1',
+ source: 'local',
+ author,
+ },
+ ] satisfies Awaited<ReturnType<SkillDeletionDeps['listSkillFiles']>>;
+ const intents: SkillUploadIntentRecord[] = [];
+ const deps: SkillDeletionDeps = {
+ beginSkillDeletion: jest.fn(async () => journal),
+ getSkillDeletion: jest.fn(async () => journal),
+ listSkillFiles: jest.fn(async () => files),
+ listSkillUploadIntents: jest.fn(async () => intents),
+ deleteBlob: jest.fn(async () => undefined),
+ deleteReservedBlob: jest.fn(async () => undefined),
+ deleteSkillFileAfterBlob: jest.fn(async (_id, fileId) => {
+ const index = files.findIndex((file) => file.file_id === fileId);
+ if (index < 0) return false;
+ files.splice(index, 1);
+ return true;
+ }),
+ finalizeSkillDeletion: jest.fn(async () => ({
+ deleted: true,
+ skillAbsent: true,
+ cleanupComplete: true,
+ failedCleanupSteps: [],
+ })),
+ completeSkillDeletion: jest.fn(async () => true),
+ reopenSkillDeletion: jest.fn(async () => undefined),
+ markSkillUploadIntentCleaned: jest.fn(async (fileId) => {
+ const intent = intents.find((row) => row.fileId === fileId);
+ if (intent) intent.cleanedAt = new Date();
+ }),
+ };
+ return { files, intents, deps };
+}
+
+it('keeps the file row when provider deletion fails and succeeds on retry', async () => {
+ const h = harness();
+ const deleteBlob = h.deps.deleteBlob as jest.Mock;
+ deleteBlob.mockRejectedValueOnce(new Error('storage unavailable'));
+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({
+ cleanupComplete: false,
+ failedSteps: ['skill_files'],
+ });
+ expect(h.files).toHaveLength(1);
+ expect(h.deps.finalizeSkillDeletion).not.toHaveBeenCalled();
+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({
+ cleanupComplete: true,
+ });
+ expect(h.files).toHaveLength(0);
+ expect(h.deps.completeSkillDeletion).toHaveBeenCalledTimes(1);
+});
+
+it('does not report completion when the claim transfers after its final refresh', async () => {
+ const h = harness();
+ h.files.length = 0;
+ (h.deps.completeSkillDeletion as jest.Mock).mockResolvedValueOnce(false);
+ await expect(runSkillDeletion(skillId, h.deps)).rejects.toBeInstanceOf(
+ SkillDeletionClaimLostError,
+ );
+ expect(h.deps.completeSkillDeletion).toHaveBeenCalledWith(skillId);
+});
+
+it('preserves idempotent absence and per-call deletion semantics', async () => {
+ const h = harness();
+ (h.deps.beginSkillDeletion as jest.Mock).mockResolvedValueOnce(null);
+ (h.deps.getSkillDeletion as jest.Mock).mockResolvedValueOnce(null);
+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({
+ found: false,
+ deleted: false,
+ skillAbsent: true,
+ cleanupComplete: true,
+ });
+
+ h.files.length = 0;
+ (h.deps.finalizeSkillDeletion as jest.Mock).mockResolvedValueOnce({
+ deleted: false,
+ skillAbsent: true,
+ cleanupComplete: true,
+ failedCleanupSteps: [],
+ });
+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({
+ found: true,
+ deleted: false,
+ skillAbsent: true,
+ cleanupComplete: true,
+ });
+});
+
+it.each(['uploading', 'committing'] as const)(
+ 'leaves an in-flight %s writer pending without deleting its blob',
+ async (state) => {
+ const h = harness();
+ h.intents.push({
+ skillId,
+ author,
+ fileId: 'file-1',
+ source: 'local',
+ storageKey: '/uploads/owner/file-1',
+ state,
+ });
+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({
+ cleanupComplete: false,
+ failedSteps: ['upload_intents'],
+ });
+ expect(h.deps.deleteBlob).not.toHaveBeenCalled();
+ expect(h.deps.finalizeSkillDeletion).not.toHaveBeenCalled();
+ },
+);
+
+it('cleans a committed historical blob after its SkillFile row was replaced', async () => {
+ const h = harness();
+ h.files.length = 0;
+ h.intents.push({
+ skillId,
+ author,
+ fileId: 'old-file',
+ source: 'local',
+ storageKey: '/uploads/owner/old-file',
+ state: 'committed',
+ });
+ const first = await runSkillDeletion(skillId.toString(), h.deps);
+ expect(first.cleanupComplete).toBe(true);
+ expect(h.deps.deleteReservedBlob).toHaveBeenCalledTimes(1);
+ expect(h.intents[0].cleanedAt).toBeDefined();
+ await runSkillDeletion(skillId.toString(), h.deps);
+ expect(h.deps.deleteReservedBlob).toHaveBeenCalledTimes(1);
+});
+
+it('deletes a current file blob only once when its committed intent points at the same upload', async () => {
+ const h = harness();
+ h.intents.push({
+ skillId,
+ author,
+ fileId: 'file-1',
+ source: 'local',
+ storageKey: '/uploads/owner/file-1',
+ state: 'committed',
+ });
+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({
+ cleanupComplete: true,
+ });
+ expect(h.deps.deleteBlob).toHaveBeenCalledTimes(1);
+ expect(h.deps.deleteReservedBlob).not.toHaveBeenCalled();
+ expect(h.intents[0].cleanedAt).toBeDefined();
+});
+
+it('resweeps abandoned intents even after an earlier clean pass', async () => {
+ const h = harness();
+ h.files.length = 0;
+ h.intents.push({
+ skillId,
+ author,
+ fileId: 'zombie',
+ source: 'local',
+ storageKey: '/uploads/owner/zombie',
+ state: 'abandoned',
+ });
+ await runSkillDeletion(skillId.toString(), h.deps);
+ await runSkillDeletion(skillId.toString(), h.deps);
+ expect(h.deps.deleteReservedBlob).toHaveBeenCalledTimes(2);
+});
+
+it('cleans intent blobs with bounded concurrency', async () => {
+ const h = harness();
+ h.files.length = 0;
+ for (let index = 0; index < 4; index++) {
+ h.intents.push({
+ skillId,
+ author,
+ fileId: `orphan-${index}`,
+ source: 'local',
+ storageKey: `/uploads/owner/orphan-${index}`,
+ state: 'abandoned',
+ });
+ }
+ let active = 0;
+ let peak = 0;
+ h.deps.deleteReservedBlob = jest.fn(async () => {
+ active++;
+ peak = Math.max(peak, active);
+ await new Promise((resolve) => setImmediate(resolve));
+ active--;
+ });
+
+ await runSkillDeletion(skillId, h.deps, 2);
+ expect(h.deps.deleteReservedBlob).toHaveBeenCalledTimes(4);
+ expect(peak).toBe(2);
+});
+
+it('does not reopen or finalize an already-completed tombstone with no late work', async () => {
+ const h = harness();
+ h.files.length = 0;
+ (h.deps.beginSkillDeletion as jest.Mock).mockResolvedValue({
+ skillId,
+ completedAt: new Date(),
+ });
+ await expect(runSkillDeletion(skillId, h.deps)).resolves.toMatchObject({
+ cleanupComplete: true,
+ skillAbsent: true,
+ });
+ expect(h.deps.reopenSkillDeletion).not.toHaveBeenCalled();
+ expect(h.deps.finalizeSkillDeletion).not.toHaveBeenCalled();
+ expect(h.deps.completeSkillDeletion).not.toHaveBeenCalled();
+});
+
+it('cold-audits an abandoned intent without rerunning completed dependent cleanup', async () => {
+ const h = harness();
+ h.files.length = 0;
+ h.intents.push({
+ skillId,
+ author,
+ fileId: 'late-write',
+ source: 'local',
+ storageKey: '/uploads/late-write',
+ state: 'abandoned',
+ cleanedAt: new Date(),
+ });
+ (h.deps.beginSkillDeletion as jest.Mock).mockResolvedValue({
+ skillId,
+ completedAt: new Date(),
+ });
+ await expect(runSkillDeletion(skillId, h.deps)).resolves.toMatchObject({
+ cleanupComplete: true,
+ });
+ expect(h.deps.deleteReservedBlob).toHaveBeenCalledTimes(1);
+ expect(h.deps.reopenSkillDeletion).not.toHaveBeenCalled();
+ expect(h.deps.finalizeSkillDeletion).not.toHaveBeenCalled();
+});
+
+it('reopens completed deletion only when a late file needs cleanup', async () => {
+ const h = harness();
+ (h.deps.beginSkillDeletion as jest.Mock).mockResolvedValue({
+ skillId,
+ completedAt: new Date(),
+ });
+ await expect(runSkillDeletion(skillId, h.deps)).resolves.toMatchObject({
+ cleanupComplete: true,
+ });
+ expect(h.deps.reopenSkillDeletion).toHaveBeenCalledWith(skillId);
+ expect(h.deps.deleteBlob).toHaveBeenCalledTimes(1);
+ expect(h.deps.finalizeSkillDeletion).toHaveBeenCalledTimes(1);
+});
+
+it('does not mutate a file row after losing the claim during its provider delete', async () => {
+ const h = harness();
+ let releaseBlob: (() => void) | undefined;
+ let startedBlob: (() => void) | undefined;
+ const started = new Promise<void>((resolve) => {
+ startedBlob = resolve;
+ });
+ (h.deps.deleteBlob as jest.Mock).mockImplementation(
+ () =>
+ new Promise<void>((resolve) => {
+ releaseBlob = resolve;
+ startedBlob?.();
+ }),
+ );
+ let lost = false;
+ const assertClaim = jest.fn(async () => {
+ if (lost) throw new SkillDeletionClaimLostError();
+ });
+ const deletion = runSkillDeletion(skillId, h.deps, 1, assertClaim);
+ await started;
+ lost = true;
+ releaseBlob?.();
+ await expect(deletion).rejects.toBeInstanceOf(SkillDeletionClaimLostError);
+ expect(h.deps.deleteSkillFileAfterBlob).not.toHaveBeenCalled();
+ expect(h.deps.finalizeSkillDeletion).not.toHaveBeenCalled();
+});
+
+it('bounds independent file cleanup while deleting each row after its blob', async () => {
+ const h = harness();
+ h.files.push({ ...h.files[0], file_id: 'file-2' }, { ...h.files[0], file_id: 'file-3' });
+ let active = 0;
+ let peak = 0;
+ (h.deps.deleteBlob as jest.Mock).mockImplementation(async () => {
+ active++;
+ peak = Math.max(peak, active);
+ await new Promise((resolve) => setImmediate(resolve));
+ active--;
+ });
+ await runSkillDeletion(skillId, h.deps, 2);
+ expect(peak).toBe(2);
+ expect(h.deps.deleteSkillFileAfterBlob).toHaveBeenCalledTimes(3);
+ expect(h.files).toHaveLength(0);
+});
diff --git a/packages/api/src/skills/deletion.ts b/packages/api/src/skills/deletion.ts
new file mode 100644
index 0000000000..cf237a2a69
--- /dev/null
+++ b/packages/api/src/skills/deletion.ts
@@ -0,0 +1,198 @@
+import { DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY } from 'librechat-data-provider';
+import type {
+ DeleteSkillCleanupStep,
+ DeleteSkillResult,
+ SkillDeletionFileRecord,
+ SkillDeletionRecord,
+ SkillUploadIntentRecord,
+} from '@librechat/data-schemas';
+import { createConcurrencyLimiter } from '~/utils/promise';
+
+export interface SkillDeletionDeps {
+ beginSkillDeletion: (id: string) => Promise<SkillDeletionRecord | null>;
+ getSkillDeletion: (id: string) => Promise<SkillDeletionRecord | null>;
+ listSkillFiles: (id: string) => Promise<SkillDeletionFileRecord[]>;
+ listSkillUploadIntents: (id: string) => Promise<SkillUploadIntentRecord[]>;
+ deleteBlob: (file: SkillDeletionFileRecord) => Promise<void>;
+ deleteReservedBlob: (intent: SkillUploadIntentRecord) => Promise<void>;
+ deleteSkillFileAfterBlob: (skillId: string, fileId: string) => Promise<boolean>;
+ finalizeSkillDeletion: (id: string) => Promise<DeleteSkillResult>;
+ completeSkillDeletion: (id: string) => Promise<boolean>;
+ reopenSkillDeletion: (id: string) => Promise<void>;
+ markSkillUploadIntentCleaned: (fileId: string) => Promise<void>;
+}
+
+export type SkillDeletionProgress = {
+ found: boolean;
+ deleted: boolean;
+ skillAbsent: boolean;
+ cleanupComplete: boolean;
+ failedSteps: DeleteSkillCleanupStep[];
+};
+
+export class SkillDeletionClaimLostError extends Error {
+ constructor() {
+ super('Skill deletion claim was lost');
+ this.name = 'SkillDeletionClaimLostError';
+ }
+}
+
+/**
+ * Every externally stored object has a persistent Mongo reference until its
+ * provider and RAG cleanup succeeds. Failures remain retryable after restart.
+ */
+export async function runSkillDeletion(
+ id: string,
+ deps: SkillDeletionDeps,
+ fileCleanupConcurrency: number = DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY,
+ assertClaim: () => Promise<void> = async () => undefined,
+ loadedJournal?: SkillDeletionRecord,
+): Promise<SkillDeletionProgress> {
+ await assertClaim();
+ const journal =
+ loadedJournal ?? (await deps.beginSkillDeletion(id)) ?? (await deps.getSkillDeletion(id));
+ if (!journal) {
+ return {
+ found: false,
+ deleted: false,
+ skillAbsent: true,
+ cleanupComplete: true,
+ failedSteps: [],
+ };
+ }
+ const failedSteps: DeleteSkillCleanupStep[] = [];
+ const [intents, files] = await Promise.all([
+ deps.listSkillUploadIntents(journal.skillId),
+ deps.listSkillFiles(id),
+ ]);
+ const requiresFullCleanup =
+ files.length > 0 ||
+ intents.some(
+ (intent) =>
+ intent.state === 'uploading' ||
+ intent.state === 'committing' ||
+ (intent.state === 'committed' && !intent.cleanedAt),
+ );
+ if (journal.completedAt && requiresFullCleanup) {
+ await assertClaim();
+ await deps.reopenSkillDeletion(journal.skillId);
+ }
+ // A writer may still finish uploading or committing after this scan. An
+ // operator can inspect and recover an intent with `recover-skill-upload`
+ // after draining writers; a timer must not infer that a slow upload is dead.
+ if (intents.some((intent) => intent.state === 'uploading' || intent.state === 'committing')) {
+ return {
+ found: true,
+ deleted: false,
+ skillAbsent: journal.completedAt != null,
+ cleanupComplete: false,
+ failedSteps: ['upload_intents'],
+ };
+ }
+ const currentFileIds = new Set(files.map((file) => file.file_id));
+ const cleanedFileIds = new Set<string>();
+ const limit = createConcurrencyLimiter(fileCleanupConcurrency);
+ const fileResults = await Promise.allSettled(
+ files.map((file) =>
+ limit(async () => {
+ await assertClaim();
+ try {
+ await deps.deleteBlob(file);
+ } catch {
+ failedSteps.push('skill_files');
+ return;
+ }
+ await assertClaim();
+ try {
+ await deps.deleteSkillFileAfterBlob(id, file.file_id);
+ cleanedFileIds.add(file.file_id);
+ } catch {
+ failedSteps.push('skill_files');
+ }
+ }),
+ ),
+ );
+ const rejectedFile = fileResults.find((result) => result.status === 'rejected');
+ if (rejectedFile?.status === 'rejected') throw rejectedFile.reason;
+
+ const intentResults = await Promise.allSettled(
+ intents.map((intent) =>
+ limit(async () => {
+ if (intent.state === 'committed') {
+ if (intent.cleanedAt) return;
+ if (currentFileIds.has(intent.fileId) && !cleanedFileIds.has(intent.fileId)) {
+ // Keep the intent with its current row for the next retry.
+ return;
+ }
+ await assertClaim();
+ try {
+ if (!cleanedFileIds.has(intent.fileId)) {
+ await deps.deleteReservedBlob(intent);
+ }
+ } catch {
+ failedSteps.push('upload_intents');
+ return;
+ }
+ await assertClaim();
+ try {
+ await deps.markSkillUploadIntentCleaned(intent.fileId);
+ } catch {
+ failedSteps.push('upload_intents');
+ }
+ return;
+ }
+ await assertClaim();
+ try {
+ await deps.deleteReservedBlob(intent);
+ // Keep abandoned intents as tombstones: a zombie upload can finish
+ // after this cleanup and must be discovered on the next sweep.
+ } catch {
+ failedSteps.push('upload_intents');
+ }
+ }),
+ ),
+ );
+ const rejectedIntent = intentResults.find((result) => result.status === 'rejected');
+ if (rejectedIntent?.status === 'rejected') throw rejectedIntent.reason;
+
+ if (failedSteps.length > 0) {
+ return {
+ found: true,
+ deleted: false,
+ skillAbsent: journal.completedAt != null,
+ cleanupComplete: false,
+ failedSteps,
+ };
+ }
+ if (journal.completedAt && !requiresFullCleanup) {
+ return {
+ found: true,
+ deleted: true,
+ skillAbsent: true,
+ cleanupComplete: true,
+ failedSteps: [],
+ };
+ }
+ await assertClaim();
+ const finalized = await deps.finalizeSkillDeletion(id);
+ if (!finalized.cleanupComplete) {
+ return {
+ found: true,
+ deleted: finalized.deleted,
+ skillAbsent: finalized.skillAbsent,
+ cleanupComplete: false,
+ failedSteps: finalized.failedCleanupSteps,
+ };
+ }
+ await assertClaim();
+ if (!(await deps.completeSkillDeletion(journal.skillId))) {
+ throw new SkillDeletionClaimLostError();
+ }
+ return {
+ found: true,
+ deleted: finalized.deleted,
+ skillAbsent: finalized.skillAbsent,
+ cleanupComplete: true,
+ failedSteps: [],
+ };
+}
diff --git a/packages/api/src/skills/deletionService.spec.ts b/packages/api/src/skills/deletionService.spec.ts
new file mode 100644
index 0000000000..3c271df4a9
--- /dev/null
+++ b/packages/api/src/skills/deletionService.spec.ts
@@ -0,0 +1,548 @@
+import type { AppConfig, SkillUploadIntentRecord } from '@librechat/data-schemas';
+import { createSkillDeletionService } from './deletionService';
+
+it('records failed cleanup attempts before releasing the claim', async () => {
+ const skillId = '507f1f77bcf86cd799439011';
+ const journal = { skillId };
+ const order: string[] = [];
+ let released: () => void = () => undefined;
+ const releaseObserved = new Promise<void>((resolve) => {
+ released = resolve;
+ });
+ const persistence = {
+ discoverUnjournaledDeletions: jest.fn(async () => 0),
+ listSkillDeletionTombstones: jest.fn(async () => [journal]),
+ claimSkillDeletion: jest.fn(async () => true),
+ refreshSkillDeletionClaim: jest.fn(async () => true),
+ noteSkillDeletionAttempt: jest.fn(async () => {
+ order.push('attempt');
+ }),
+ releaseSkillDeletionClaim: jest.fn(async (): Promise<void> => {
+ order.push('release');
+ released();
+ }),
+ beginSkillDeletion: jest.fn(async () => journal),
+ getSkillDeletion: jest.fn(async (): Promise<typeof journal | null> => journal),
+ listSkillFiles: jest.fn(async () => {
+ throw new Error('storage unavailable');
+ }),
+ listSkillUploadIntents: jest.fn(async () => []),
+ deleteSkillFileAfterBlob: jest.fn(async () => true),
+ finalizeSkillDeletion: jest.fn(async () => ({
+ deleted: true,
+ skillAbsent: true,
+ cleanupComplete: true,
+ failedCleanupSteps: [],
+ })),
+ completeSkillDeletion: jest.fn(async () => true),
+ reopenSkillDeletion: jest.fn(async () => undefined),
+ markSkillUploadIntentCleaned: jest.fn(async () => undefined),
+ listAbandonedSkillUploadIntents: jest.fn(async (): Promise<SkillUploadIntentRecord[]> => []),
+ claimAbandonedSkillUploadIntent: jest.fn(async () => true),
+ refreshAbandonedSkillUploadIntentClaim: jest.fn(async () => true),
+ noteAbandonedSkillUploadIntentAttempt: jest.fn(async () => undefined),
+ releaseAbandonedSkillUploadIntentClaim: jest.fn(async () => undefined),
+ scheduleSkillDeletionRecheck: jest.fn(async () => true),
+ scheduleAbandonedSkillUploadIntentRecheck: jest.fn(async () => true),
+ skillFileExistsForIntent: jest.fn(async () => false),
+ };
+ const service = createSkillDeletionService({
+ persistence,
+ loadConfig: jest.fn(
+ async () => ({ skillDeletion: { finalizationEnabled: true } }) as unknown as AppConfig,
+ ),
+ getStrategyFunctions: jest.fn(() => ({})),
+ runAsSystem: async (callback) => callback(),
+ logger: { error: jest.fn(), warn: jest.fn() },
+ });
+
+ const sweep = service.startSkillDeletionSweep({ sweepIntervalMs: 60_000, claimLeaseMs: 120_000 });
+ await releaseObserved;
+ sweep.stop();
+ expect(order).toEqual(['attempt', 'release']);
+ expect(persistence.beginSkillDeletion).toHaveBeenCalledTimes(1);
+ expect(persistence.getSkillDeletion).not.toHaveBeenCalled();
+ expect(persistence.noteSkillDeletionAttempt).toHaveBeenCalledWith(skillId);
+});
+
+it.each([false, true])(
+ 'reconciles an abandoned intent on an active Skill while preserving a live file row: row=%s',
+ async (rowExists) => {
+ const skillId = '507f1f77bcf86cd799439011';
+ const author = '507f1f77bcf86cd799439012';
+ const intent = {
+ skillId,
+ fileId: 'abandoned-file',
+ source: 'local',
+ storageKey: '/uploads/abandoned-file',
+ author,
+ state: 'abandoned' as const,
+ };
+ let released: () => void = () => undefined;
+ const releaseObserved = new Promise<void>((resolve) => {
+ released = resolve;
+ });
+ const deleteFile = jest.fn(async () => undefined);
+ const persistence = {
+ discoverUnjournaledDeletions: jest.fn(async () => 0),
+ listSkillDeletionTombstones: jest.fn(async () => []),
+ claimSkillDeletion: jest.fn(async () => true),
+ refreshSkillDeletionClaim: jest.fn(async () => true),
+ noteSkillDeletionAttempt: jest.fn(async () => undefined),
+ releaseSkillDeletionClaim: jest.fn(async () => undefined),
+ beginSkillDeletion: jest.fn(async () => null),
+ getSkillDeletion: jest.fn(async () => null),
+ listSkillFiles: jest.fn(async () => []),
+ listSkillUploadIntents: jest.fn(async () => []),
+ deleteSkillFileAfterBlob: jest.fn(async () => true),
+ finalizeSkillDeletion: jest.fn(async () => ({
+ deleted: false,
+ skillAbsent: true,
+ cleanupComplete: true,
+ failedCleanupSteps: [],
+ })),
+ completeSkillDeletion: jest.fn(async () => true),
+ reopenSkillDeletion: jest.fn(async () => undefined),
+ markSkillUploadIntentCleaned: jest.fn(async () => undefined),
+ listAbandonedSkillUploadIntents: jest.fn(async () => [intent]),
+ claimAbandonedSkillUploadIntent: jest.fn(async () => true),
+ refreshAbandonedSkillUploadIntentClaim: jest.fn(async () => true),
+ noteAbandonedSkillUploadIntentAttempt: jest.fn(async () => undefined),
+ releaseAbandonedSkillUploadIntentClaim: jest.fn(async () => released()),
+ scheduleSkillDeletionRecheck: jest.fn(async () => true),
+ scheduleAbandonedSkillUploadIntentRecheck: jest.fn(async () => true),
+ skillFileExistsForIntent: jest.fn(async () => rowExists),
+ };
+ const service = createSkillDeletionService({
+ persistence,
+ loadConfig: jest.fn(async () => undefined),
+ getStrategyFunctions: jest.fn(() => ({ deleteFile })),
+ runAsSystem: async (callback) => callback(),
+ logger: { error: jest.fn(), warn: jest.fn() },
+ });
+
+ const sweep = service.startSkillDeletionSweep();
+ await releaseObserved;
+ sweep.stop();
+ expect(deleteFile).toHaveBeenCalledTimes(rowExists ? 0 : 1);
+ expect(persistence.markSkillUploadIntentCleaned).toHaveBeenCalledTimes(rowExists ? 0 : 1);
+ expect(persistence.scheduleAbandonedSkillUploadIntentRecheck).toHaveBeenCalledWith(
+ intent.fileId,
+ expect.any(String),
+ 3_600_000,
+ );
+ expect(persistence.noteAbandonedSkillUploadIntentAttempt).toHaveBeenCalledWith(intent.fileId);
+ },
+);
+
+function claimedServiceHarness() {
+ const skillId = '507f1f77bcf86cd799439011';
+ const journal = { skillId, completedAt: undefined as Date | undefined };
+ const files = [
+ {
+ file_id: 'file-1',
+ filepath: '/uploads/file-1',
+ source: 'local',
+ author: '507f1f77bcf86cd799439012',
+ },
+ ];
+ let held = false;
+ const persistence = {
+ discoverUnjournaledDeletions: jest.fn(async () => 0),
+ listSkillDeletionTombstones: jest.fn(async () => [journal]),
+ claimSkillDeletion: jest.fn(async () => {
+ if (held) return false;
+ held = true;
+ return true;
+ }),
+ refreshSkillDeletionClaim: jest.fn(async () => held),
+ noteSkillDeletionAttempt: jest.fn(async () => undefined),
+ releaseSkillDeletionClaim: jest.fn(async (): Promise<void> => {
+ held = false;
+ }),
+ scheduleSkillDeletionRecheck: jest.fn(async () => true),
+ beginSkillDeletion: jest.fn(async () => journal),
+ getSkillDeletion: jest.fn(async (): Promise<typeof journal | null> => journal),
+ listSkillFiles: jest.fn(async () => files),
+ listSkillUploadIntents: jest.fn(async () => []),
+ deleteSkillFileAfterBlob: jest.fn(async () => {
+ files.length = 0;
+ return true;
+ }),
+ finalizeSkillDeletion: jest.fn(async () => ({
+ deleted: true,
+ skillAbsent: true,
+ cleanupComplete: true,
+ failedCleanupSteps: [],
+ })),
+ completeSkillDeletion: jest.fn(async () => {
+ journal.completedAt = new Date();
+ return true;
+ }),
+ reopenSkillDeletion: jest.fn(async () => undefined),
+ markSkillUploadIntentCleaned: jest.fn(async () => undefined),
+ listAbandonedSkillUploadIntents: jest.fn(async (): Promise<SkillUploadIntentRecord[]> => []),
+ claimAbandonedSkillUploadIntent: jest.fn(async () => true),
+ refreshAbandonedSkillUploadIntentClaim: jest.fn(async () => true),
+ noteAbandonedSkillUploadIntentAttempt: jest.fn(async () => undefined),
+ releaseAbandonedSkillUploadIntentClaim: jest.fn(async () => undefined),
+ scheduleAbandonedSkillUploadIntentRecheck: jest.fn(async () => true),
+ skillFileExistsForIntent: jest.fn(async () => false),
+ };
+ const deleteFile = jest.fn(async (): Promise<void> => undefined);
+ const loadConfig = jest.fn(
+ async (): Promise<AppConfig | undefined> =>
+ ({ skillDeletion: { finalizationEnabled: true } }) as unknown as AppConfig,
+ );
+ const logger = { error: jest.fn(), warn: jest.fn() };
+ const service = createSkillDeletionService({
+ persistence,
+ loadConfig,
+ getStrategyFunctions: jest.fn(() => ({ deleteFile })),
+ runAsSystem: async (callback) => callback(),
+ logger,
+ });
+ return { skillId, persistence, deleteFile, loadConfig, logger, service };
+}
+
+it('marks deletion but cannot finalize until the writer-drain gate is enabled', async () => {
+ const h = claimedServiceHarness();
+ h.loadConfig.mockResolvedValue({
+ skillDeletion: { finalizationEnabled: false },
+ } as unknown as AppConfig);
+
+ const pending = await h.service.deleteSkillById(h.skillId);
+ expect(pending).toMatchObject({ found: true, deleted: false, cleanupComplete: false });
+ expect(h.persistence.beginSkillDeletion).toHaveBeenCalledWith(h.skillId);
+ expect(h.persistence.claimSkillDeletion).not.toHaveBeenCalled();
+ expect(h.persistence.finalizeSkillDeletion).not.toHaveBeenCalled();
+
+ h.loadConfig.mockResolvedValue({
+ skillDeletion: { finalizationEnabled: true },
+ } as unknown as AppConfig);
+ const completed = await h.service.deleteSkillById(h.skillId);
+ expect(completed.cleanupComplete).toBe(true);
+ expect(h.persistence.finalizeSkillDeletion).toHaveBeenCalledTimes(1);
+});
+
+it('returns pending when the claim transfers between its final check and completion write', async () => {
+ const h = claimedServiceHarness();
+ h.persistence.completeSkillDeletion.mockResolvedValueOnce(false);
+
+ const progress = await h.service.deleteSkillById(h.skillId);
+ expect(progress).toMatchObject({ found: true, deleted: false, cleanupComplete: false });
+ expect(h.persistence.completeSkillDeletion).toHaveBeenCalledWith(h.skillId, expect.any(String));
+ expect(h.persistence.releaseSkillDeletionClaim).toHaveBeenCalledTimes(1);
+});
+
+it('processes independent deletion tombstones with bounded concurrency', async () => {
+ const h = claimedServiceHarness();
+ h.persistence.listSkillDeletionTombstones.mockResolvedValue(
+ ['507f1f77bcf86cd799439011', '507f1f77bcf86cd799439012', '507f1f77bcf86cd799439013'].map(
+ (skillId) => ({ skillId, completedAt: undefined }),
+ ),
+ );
+ h.persistence.claimSkillDeletion.mockResolvedValue(true);
+ h.persistence.refreshSkillDeletionClaim.mockResolvedValue(true);
+ h.persistence.listSkillFiles.mockImplementation(async () => [
+ {
+ file_id: 'file-1',
+ filepath: '/uploads/file-1',
+ source: 'local',
+ author: '507f1f77bcf86cd799439012',
+ },
+ {
+ file_id: 'file-2',
+ filepath: '/uploads/file-2',
+ source: 'local',
+ author: '507f1f77bcf86cd799439012',
+ },
+ ]);
+ h.persistence.deleteSkillFileAfterBlob.mockResolvedValue(true);
+ h.loadConfig.mockResolvedValue({
+ skillDeletion: { finalizationEnabled: true },
+ fileConfig: { skills: { deletionCleanupConcurrency: 2 } },
+ } as unknown as AppConfig);
+ let active = 0;
+ let peak = 0;
+ let allCompleted!: () => void;
+ const done = new Promise<void>((resolve) => {
+ allCompleted = resolve;
+ });
+ let completed = 0;
+ h.persistence.releaseSkillDeletionClaim.mockImplementation(async () => {
+ completed++;
+ if (completed === 3) allCompleted();
+ });
+ h.deleteFile.mockImplementation(async () => {
+ active++;
+ peak = Math.max(peak, active);
+ await new Promise<void>((resolve) => setTimeout(resolve, 10));
+ active--;
+ });
+
+ const sweep = h.service.startSkillDeletionSweep();
+ try {
+ await done;
+ expect(peak).toBe(2);
+ expect(h.deleteFile).toHaveBeenCalledTimes(6);
+ expect(h.persistence.finalizeSkillDeletion).toHaveBeenCalledTimes(3);
+ expect(h.loadConfig).toHaveBeenCalledTimes(1);
+ } finally {
+ sweep.stop();
+ }
+});
+
+it('uses the configured claim lease for direct deletion', async () => {
+ const h = claimedServiceHarness();
+ h.loadConfig.mockResolvedValue({
+ skillDeletion: { finalizationEnabled: true, claimLeaseMs: 9_000 },
+ } as unknown as AppConfig);
+ await h.service.deleteSkillById(h.skillId);
+ expect(h.persistence.claimSkillDeletion).toHaveBeenCalledWith(
+ h.skillId,
+ expect.any(String),
+ 9_000,
+ );
+ expect(h.loadConfig).toHaveBeenCalledTimes(1);
+ expect(h.persistence.beginSkillDeletion).toHaveBeenCalledTimes(1);
+ expect(h.persistence.getSkillDeletion).not.toHaveBeenCalled();
+});
+
+it('does not log provider metadata when abandoned-intent cleanup fails', async () => {
+ const h = claimedServiceHarness();
+ const fileId = 'abandoned-file';
+ h.persistence.listSkillDeletionTombstones.mockResolvedValue([]);
+ h.persistence.listAbandonedSkillUploadIntents.mockResolvedValue([
+ {
+ skillId: h.skillId,
+ fileId,
+ source: 'local',
+ storageKey: '/uploads/abandoned-file',
+ author: '507f1f77bcf86cd799439012',
+ state: 'abandoned',
+ },
+ ]);
+ h.persistence.getSkillDeletion.mockResolvedValue(null);
+ h.deleteFile.mockRejectedValue(
+ Object.assign(new Error('signed URL: secret-token'), {
+ status: 403,
+ request: { url: 'https://storage.example/object?signature=secret-token' },
+ response: { headers: { authorization: 'secret-token' } },
+ }),
+ );
+ let released!: () => void;
+ const done = new Promise<void>((resolve) => {
+ released = resolve;
+ });
+ h.persistence.releaseAbandonedSkillUploadIntentClaim.mockImplementation(async () => {
+ released();
+ return undefined;
+ });
+
+ const sweep = h.service.startSkillDeletionSweep();
+ await done;
+ sweep.stop();
+
+ expect(h.logger.error).toHaveBeenCalledWith(
+ `[skill upload intent] Cleanup failed for ${fileId}:`,
+ { type: 'Error', status: 403 },
+ );
+ expect(JSON.stringify(h.logger.error.mock.calls)).not.toContain('secret-token');
+});
+
+it('bounds independent abandoned-intent cleanup without serializing the batch', async () => {
+ const h = claimedServiceHarness();
+ h.persistence.listSkillDeletionTombstones.mockResolvedValue([]);
+ h.persistence.listAbandonedSkillUploadIntents.mockResolvedValue(
+ ['file-1', 'file-2', 'file-3'].map((fileId) => ({
+ skillId: h.skillId,
+ fileId,
+ source: 'local',
+ storageKey: `/uploads/${fileId}`,
+ author: '507f1f77bcf86cd799439012',
+ state: 'abandoned' as const,
+ })),
+ );
+ h.persistence.getSkillDeletion.mockResolvedValue(null);
+ h.loadConfig.mockResolvedValue({
+ fileConfig: { skills: { deletionCleanupConcurrency: 2 } },
+ } as AppConfig);
+ let active = 0;
+ let peak = 0;
+ let started = 0;
+ let completed = 0;
+ const releases: Array<() => void> = [];
+ let twoStarted!: () => void;
+ let threeStarted!: () => void;
+ let allReleased!: () => void;
+ const firstPair = new Promise<void>((resolve) => {
+ twoStarted = resolve;
+ });
+ const third = new Promise<void>((resolve) => {
+ threeStarted = resolve;
+ });
+ const done = new Promise<void>((resolve) => {
+ allReleased = resolve;
+ });
+ h.deleteFile.mockImplementation(async () => {
+ active++;
+ peak = Math.max(peak, active);
+ started++;
+ if (started === 2) twoStarted();
+ if (started === 3) threeStarted();
+ await new Promise<void>((resolve) => releases.push(resolve));
+ active--;
+ });
+ h.persistence.releaseAbandonedSkillUploadIntentClaim.mockImplementation(async () => {
+ completed++;
+ if (completed === 3) allReleased();
+ return undefined;
+ });
+
+ const sweep = h.service.startSkillDeletionSweep();
+ try {
+ await firstPair;
+ expect(h.deleteFile).toHaveBeenCalledTimes(2);
+ releases[0]();
+ await third;
+ expect(peak).toBe(2);
+ for (const release of releases.slice(1)) release();
+ await done;
+ expect(h.persistence.markSkillUploadIntentCleaned).toHaveBeenCalledTimes(3);
+ expect(h.loadConfig).toHaveBeenCalledTimes(2);
+ } finally {
+ sweep.stop();
+ }
+});
+
+it('defers tombstone-owned abandoned intents without deleting their blobs again', async () => {
+ const h = claimedServiceHarness();
+ const intent = {
+ skillId: h.skillId,
+ fileId: 'tombstone-owned',
+ source: 'local',
+ storageKey: '/uploads/tombstone-owned',
+ author: '507f1f77bcf86cd799439012',
+ state: 'abandoned' as const,
+ };
+ h.persistence.listSkillDeletionTombstones.mockResolvedValue([]);
+ h.persistence.listAbandonedSkillUploadIntents.mockResolvedValue([intent]);
+ let released: (() => void) | undefined;
+ const done = new Promise<void>((resolve) => {
+ released = resolve;
+ });
+ h.persistence.releaseAbandonedSkillUploadIntentClaim.mockImplementation(async () => {
+ released?.();
+ });
+ const sweep = h.service.startSkillDeletionSweep();
+ await done;
+ sweep.stop();
+ expect(h.persistence.scheduleAbandonedSkillUploadIntentRecheck).toHaveBeenCalledWith(
+ intent.fileId,
+ expect.any(String),
+ 3_600_000,
+ );
+ expect(h.deleteFile).not.toHaveBeenCalled();
+});
+
+it('returns pending to a direct caller while the sweep owns the same Skill claim', async () => {
+ const h = claimedServiceHarness();
+ let releasedClaim: (() => void) | undefined;
+ const released = new Promise<void>((resolve) => {
+ releasedClaim = resolve;
+ });
+ h.persistence.releaseSkillDeletionClaim.mockImplementation(async () => {
+ releasedClaim?.();
+ });
+ let releaseBlob: (() => void) | undefined;
+ let startedBlob: (() => void) | undefined;
+ const started = new Promise<void>((resolve) => {
+ startedBlob = resolve;
+ });
+ h.deleteFile.mockImplementation(
+ () =>
+ new Promise<void>((resolve) => {
+ releaseBlob = resolve;
+ startedBlob?.();
+ }),
+ );
+ const sweep = h.service.startSkillDeletionSweep();
+ await started;
+ const pending = await h.service.deleteSkillById(h.skillId);
+ expect(pending).toMatchObject({ found: true, cleanupComplete: false });
+ expect(h.deleteFile).toHaveBeenCalledTimes(1);
+ releaseBlob?.();
+ await released;
+ sweep.stop();
+ expect(h.persistence.finalizeSkillDeletion).toHaveBeenCalledTimes(1);
+});
+
+it('treats a deferred completed tombstone as an idempotent direct deletion', async () => {
+ const h = claimedServiceHarness();
+ const completed = { skillId: h.skillId, completedAt: new Date() };
+ h.persistence.beginSkillDeletion.mockResolvedValue(completed);
+ h.persistence.getSkillDeletion.mockResolvedValue(completed);
+ h.persistence.claimSkillDeletion.mockResolvedValue(false);
+ await expect(h.service.deleteSkillById(h.skillId)).resolves.toMatchObject({
+ found: true,
+ skillAbsent: true,
+ cleanupComplete: true,
+ });
+ expect(h.deleteFile).not.toHaveBeenCalled();
+});
+
+it('preserves a completed deletion result when finalization is later disabled', async () => {
+ const h = claimedServiceHarness();
+ const completed = { skillId: h.skillId, completedAt: new Date() };
+ h.persistence.beginSkillDeletion.mockResolvedValue(completed);
+ h.loadConfig.mockResolvedValue({
+ skillDeletion: { finalizationEnabled: false },
+ } as unknown as AppConfig);
+
+ await expect(h.service.deleteSkillById(h.skillId)).resolves.toMatchObject({
+ found: true,
+ deleted: true,
+ skillAbsent: true,
+ cleanupComplete: true,
+ });
+ await expect(h.service.deleteSkillForLegacy(h.skillId)).resolves.toMatchObject({
+ deleted: true,
+ skillAbsent: true,
+ cleanupComplete: true,
+ });
+ expect(h.persistence.claimSkillDeletion).not.toHaveBeenCalled();
+ expect(h.persistence.finalizeSkillDeletion).not.toHaveBeenCalled();
+});
+
+it('does not delete a file row after the heartbeat reports a lost claim', async () => {
+ jest.useFakeTimers();
+ try {
+ const h = claimedServiceHarness();
+ let releaseBlob: (() => void) | undefined;
+ let startedBlob: (() => void) | undefined;
+ const started = new Promise<void>((resolve) => {
+ startedBlob = resolve;
+ });
+ h.deleteFile.mockImplementation(
+ () =>
+ new Promise<void>((resolve) => {
+ releaseBlob = resolve;
+ startedBlob?.();
+ }),
+ );
+ h.persistence.refreshSkillDeletionClaim
+ .mockResolvedValueOnce(true)
+ .mockResolvedValueOnce(true)
+ .mockResolvedValueOnce(false);
+ const deletion = h.service.deleteSkillById(h.skillId);
+ await started;
+ await jest.advanceTimersByTimeAsync(30_000);
+ releaseBlob?.();
+ await expect(deletion).resolves.toMatchObject({ found: true, cleanupComplete: false });
+ expect(h.persistence.deleteSkillFileAfterBlob).not.toHaveBeenCalled();
+ expect(h.persistence.finalizeSkillDeletion).not.toHaveBeenCalled();
+ } finally {
+ jest.useRealTimers();
+ }
+});
diff --git a/packages/api/src/skills/deletionService.ts b/packages/api/src/skills/deletionService.ts
new file mode 100644
index 0000000000..d6a00359b1
--- /dev/null
+++ b/packages/api/src/skills/deletionService.ts
@@ -0,0 +1,474 @@
+import { randomUUID } from 'node:crypto';
+import { DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY } from 'librechat-data-provider';
+import type {
+ AppConfig,
+ DeleteSkillResult,
+ SkillDeletionRecord,
+ SkillUploadIntentRecord,
+} from '@librechat/data-schemas';
+import type { SkillDeletionDeps, SkillDeletionProgress } from './deletion';
+import type { ServerRequest, StrategyFunctions } from '~/types';
+import { runSkillDeletion, SkillDeletionClaimLostError } from './deletion';
+import { createConcurrencyLimiter } from '~/utils/promise';
+import { getStorageMetadata } from '~/storage/metadata';
+import { getSafeErrorMetadata } from '~/utils/errors';
+
+type Persistence = Omit<
+ SkillDeletionDeps,
+ 'deleteBlob' | 'deleteReservedBlob' | 'completeSkillDeletion'
+> & {
+ completeSkillDeletion: (id: string, owner: string) => Promise<boolean>;
+ discoverUnjournaledDeletions: (limit: number) => Promise<number>;
+ listSkillDeletionTombstones: (limit: number) => Promise<SkillDeletionRecord[]>;
+ claimSkillDeletion: (id: string, owner: string, leaseMs: number) => Promise<boolean>;
+ refreshSkillDeletionClaim: (id: string, owner: string, leaseMs: number) => Promise<boolean>;
+ noteSkillDeletionAttempt: (id: string) => Promise<void>;
+ releaseSkillDeletionClaim: (id: string, owner: string) => Promise<void>;
+ scheduleSkillDeletionRecheck: (id: string, owner: string, delayMs: number) => Promise<boolean>;
+ listAbandonedSkillUploadIntents: (limit: number) => Promise<SkillUploadIntentRecord[]>;
+ claimAbandonedSkillUploadIntent: (
+ fileId: string,
+ owner: string,
+ leaseMs: number,
+ ) => Promise<boolean>;
+ refreshAbandonedSkillUploadIntentClaim: (
+ fileId: string,
+ owner: string,
+ leaseMs: number,
+ ) => Promise<boolean>;
+ noteAbandonedSkillUploadIntentAttempt: (fileId: string) => Promise<void>;
+ releaseAbandonedSkillUploadIntentClaim: (fileId: string, owner: string) => Promise<void>;
+ scheduleAbandonedSkillUploadIntentRecheck: (
+ fileId: string,
+ owner: string,
+ delayMs: number,
+ ) => Promise<boolean>;
+ skillFileExistsForIntent: (skillId: string, fileId: string) => Promise<boolean>;
+};
+
+interface SkillDeletionServiceDeps {
+ persistence: Persistence;
+ loadConfig: () => Promise<AppConfig | undefined>;
+ getStrategyFunctions: (source: string) => Partial<StrategyFunctions>;
+ runAsSystem: <T>(callback: () => Promise<T>) => Promise<T>;
+ logger: Pick<Console, 'error' | 'warn'>;
+}
+
+export interface SkillDeletionSweepOptions {
+ sweepIntervalMs: number;
+ claimLeaseMs: number;
+ batchSize: number;
+}
+
+export interface SkillDeletionService {
+ deleteSkillById: (skillId: string) => Promise<SkillDeletionProgress>;
+ deleteSkillForLegacy: (
+ skillId: string,
+ ) => Promise<DeleteSkillResult & { blobCleanupManaged: boolean }>;
+ startSkillDeletionSweep: (options?: Partial<SkillDeletionSweepOptions>) => {
+ stop: () => void;
+ runOnce: () => Promise<void>;
+ };
+}
+
+type CleanupLimiter = ReturnType<typeof createConcurrencyLimiter>;
+
+/** The legacy server supplies storage, database, and context dependencies; it owns no worker logic. */
+export function createSkillDeletionService(deps: SkillDeletionServiceDeps): SkillDeletionService {
+ async function deleteStoredBlob(
+ file: {
+ filepath: string;
+ source: string;
+ storageKey?: string | null;
+ storageRegion?: string | null;
+ author: string;
+ tenantId?: string;
+ },
+ config: AppConfig | undefined,
+ ) {
+ const strategy = deps.getStrategyFunctions(file.source);
+ if (!strategy.deleteFile) {
+ throw new Error(`Skill storage backend ${file.source} has no deletion strategy`);
+ }
+ const ownerId = file.author;
+ await strategy.deleteFile(
+ {
+ config,
+ user: { id: ownerId, _id: ownerId, tenantId: file.tenantId },
+ } as unknown as ServerRequest,
+ {
+ filepath: file.filepath,
+ storageKey: file.storageKey,
+ storageRegion: file.storageRegion,
+ user: ownerId,
+ tenantId: file.tenantId,
+ },
+ );
+ }
+
+ function pendingDeletion(found: boolean): SkillDeletionProgress {
+ return {
+ found,
+ deleted: false,
+ skillAbsent: !found,
+ cleanupComplete: !found,
+ failedSteps: [],
+ };
+ }
+
+ function completedDeletion(): SkillDeletionProgress {
+ return {
+ found: true,
+ deleted: true,
+ skillAbsent: true,
+ cleanupComplete: true,
+ failedSteps: [],
+ };
+ }
+
+ /** Each request or sweep item owns a distinct claim; same-owner reentry would bypass exclusion. */
+ async function deleteWithClaim(
+ skillId: string,
+ claimLeaseMs: number,
+ loadedConfig: AppConfig | undefined | null = null,
+ limitBlob?: CleanupLimiter,
+ ): Promise<SkillDeletionProgress> {
+ const journal =
+ (await deps.persistence.beginSkillDeletion(skillId)) ??
+ (await deps.persistence.getSkillDeletion(skillId));
+ if (!journal) return pendingDeletion(false);
+ const config = loadedConfig === null ? await deps.loadConfig() : loadedConfig;
+ if (config?.skillDeletion?.finalizationEnabled !== true) {
+ return journal.completedAt ? completedDeletion() : pendingDeletion(true);
+ }
+ const owner = randomUUID();
+ if (!(await deps.persistence.claimSkillDeletion(skillId, owner, claimLeaseMs))) {
+ // A completed tombstone may be deliberately deferred until its next cold audit.
+ // Re-read it: another worker could have reopened it after the first lookup.
+ if ((await deps.persistence.getSkillDeletion(skillId))?.completedAt) {
+ return completedDeletion();
+ }
+ return pendingDeletion(true);
+ }
+ let lost = false;
+ const assertClaim = async () => {
+ if (lost) throw new SkillDeletionClaimLostError();
+ try {
+ const retained = await deps.persistence.refreshSkillDeletionClaim(
+ skillId,
+ owner,
+ claimLeaseMs,
+ );
+ if (!retained) {
+ lost = true;
+ throw new SkillDeletionClaimLostError();
+ }
+ } catch (error) {
+ lost = true;
+ throw error;
+ }
+ };
+ const heartbeat = setInterval(() => {
+ void assertClaim().catch((error) =>
+ deps.logger.error('[skill deletion] Claim heartbeat failed:', getSafeErrorMetadata(error)),
+ );
+ }, claimLeaseMs / 4);
+ heartbeat.unref?.();
+ try {
+ const progress = await runSkillDeletion(
+ skillId,
+ {
+ ...deps.persistence,
+ completeSkillDeletion: (id) => deps.persistence.completeSkillDeletion(id, owner),
+ deleteBlob: (file) =>
+ limitBlob
+ ? limitBlob(() => deleteStoredBlob(file, config ?? undefined))
+ : deleteStoredBlob(file, config ?? undefined),
+ deleteReservedBlob: (intent) => {
+ const deleteIntentBlob = () =>
+ deleteStoredBlob(
+ {
+ filepath: intent.storageKey,
+ source: intent.source,
+ ...getStorageMetadata({ filepath: intent.storageKey, source: intent.source }),
+ author: intent.author,
+ tenantId: intent.tenantId,
+ },
+ config ?? undefined,
+ );
+ return limitBlob ? limitBlob(deleteIntentBlob) : deleteIntentBlob();
+ },
+ },
+ config?.fileConfig?.skills?.deletionCleanupConcurrency,
+ assertClaim,
+ journal,
+ );
+ if (progress.cleanupComplete) {
+ await assertClaim();
+ const scheduled = await deps.persistence.scheduleSkillDeletionRecheck(
+ skillId,
+ owner,
+ config?.skillDeletion?.recheckIntervalMs ?? 3_600_000,
+ );
+ if (!scheduled) {
+ lost = true;
+ throw new SkillDeletionClaimLostError();
+ }
+ }
+ return progress;
+ } catch (error) {
+ if (error instanceof SkillDeletionClaimLostError) return pendingDeletion(true);
+ throw error;
+ } finally {
+ clearInterval(heartbeat);
+ if (!lost) {
+ try {
+ await assertClaim();
+ await deps.persistence.noteSkillDeletionAttempt(skillId);
+ } catch (error) {
+ deps.logger.error(
+ `[skill deletion] Attempt timestamp failed for ${skillId}:`,
+ getSafeErrorMetadata(error),
+ );
+ }
+ }
+ try {
+ await deps.persistence.releaseSkillDeletionClaim(skillId, owner);
+ } catch (error) {
+ deps.logger.error(
+ `[skill deletion] Claim release failed for ${skillId}:`,
+ getSafeErrorMetadata(error),
+ );
+ }
+ }
+ }
+
+ async function deleteSkillById(skillId: string) {
+ const config = await deps.loadConfig();
+ return deleteWithClaim(skillId, config?.skillDeletion?.claimLeaseMs ?? 120_000, config);
+ }
+
+ async function deleteSkillForLegacy(skillId: string) {
+ const progress = await deleteSkillById(skillId);
+ return {
+ deleted: progress.deleted,
+ skillAbsent: progress.skillAbsent,
+ cleanupComplete: progress.cleanupComplete,
+ failedCleanupSteps: progress.failedSteps,
+ blobCleanupManaged: true,
+ };
+ }
+
+ /** Completed tombstones remain eligible for cold audits: delayed writers may surface later. */
+ function startSkillDeletionSweep(options: Partial<SkillDeletionSweepOptions> = {}) {
+ const { sweepIntervalMs = 60_000, claimLeaseMs = 120_000, batchSize = 100 } = options;
+ let running = false;
+ async function sweepAbandonedIntents() {
+ const intents = await deps.persistence.listAbandonedSkillUploadIntents(batchSize);
+ if (intents.length === 0) return;
+ const config = await deps.loadConfig();
+ const recheckIntervalMs = config?.skillDeletion?.recheckIntervalMs ?? 3_600_000;
+ const limit = createConcurrencyLimiter(
+ config?.fileConfig?.skills?.deletionCleanupConcurrency ??
+ DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY,
+ );
+ const results = await Promise.allSettled(
+ intents.map((intent) =>
+ limit(async () => {
+ const { fileId } = intent;
+ const owner = randomUUID();
+ if (
+ !(await deps.persistence.claimAbandonedSkillUploadIntent(fileId, owner, claimLeaseMs))
+ ) {
+ return;
+ }
+ let lost = false;
+ const assertClaim = async () => {
+ if (lost) throw new SkillDeletionClaimLostError();
+ try {
+ const retained = await deps.persistence.refreshAbandonedSkillUploadIntentClaim(
+ fileId,
+ owner,
+ claimLeaseMs,
+ );
+ if (!retained) {
+ lost = true;
+ throw new SkillDeletionClaimLostError();
+ }
+ } catch (error) {
+ lost = true;
+ throw error;
+ }
+ };
+ const heartbeat = setInterval(() => {
+ void assertClaim().catch((error) =>
+ deps.logger.error(
+ '[skill upload intent] Claim heartbeat failed:',
+ getSafeErrorMetadata(error),
+ ),
+ );
+ }, claimLeaseMs / 4);
+ heartbeat.unref?.();
+ try {
+ const skillId = intent.skillId;
+ await assertClaim();
+ if (await deps.persistence.getSkillDeletion(skillId)) {
+ // Whole-Skill deletion owns intents for this Skill.
+ await assertClaim();
+ if (
+ !(await deps.persistence.scheduleAbandonedSkillUploadIntentRecheck(
+ fileId,
+ owner,
+ recheckIntervalMs,
+ ))
+ ) {
+ lost = true;
+ throw new SkillDeletionClaimLostError();
+ }
+ return;
+ }
+ await assertClaim();
+ if (await deps.persistence.skillFileExistsForIntent(skillId, fileId)) {
+ deps.logger.warn(
+ `[skill upload intent] ${fileId} still has a file row; retaining blob`,
+ );
+ await assertClaim();
+ if (
+ !(await deps.persistence.scheduleAbandonedSkillUploadIntentRecheck(
+ fileId,
+ owner,
+ recheckIntervalMs,
+ ))
+ ) {
+ lost = true;
+ throw new SkillDeletionClaimLostError();
+ }
+ return;
+ }
+ await assertClaim();
+ await deleteStoredBlob(
+ {
+ filepath: intent.storageKey,
+ source: intent.source,
+ ...getStorageMetadata({ filepath: intent.storageKey, source: intent.source }),
+ author: intent.author,
+ tenantId: intent.tenantId,
+ },
+ config,
+ );
+ await assertClaim();
+ await deps.persistence.markSkillUploadIntentCleaned(fileId);
+ await assertClaim();
+ if (
+ !(await deps.persistence.scheduleAbandonedSkillUploadIntentRecheck(
+ fileId,
+ owner,
+ recheckIntervalMs,
+ ))
+ ) {
+ lost = true;
+ throw new SkillDeletionClaimLostError();
+ }
+ } catch (error) {
+ deps.logger.error(
+ `[skill upload intent] Cleanup failed for ${fileId}:`,
+ getSafeErrorMetadata(error),
+ );
+ } finally {
+ clearInterval(heartbeat);
+ if (!lost) {
+ try {
+ await assertClaim();
+ await deps.persistence.noteAbandonedSkillUploadIntentAttempt(fileId);
+ } catch (error) {
+ deps.logger.error(
+ `[skill upload intent] Attempt timestamp failed for ${fileId}:`,
+ getSafeErrorMetadata(error),
+ );
+ }
+ }
+ try {
+ await deps.persistence.releaseAbandonedSkillUploadIntentClaim(fileId, owner);
+ } catch (error) {
+ deps.logger.error(
+ `[skill upload intent] Claim release failed for ${fileId}:`,
+ getSafeErrorMetadata(error),
+ );
+ }
+ }
+ }),
+ ),
+ );
+ for (const result of results) {
+ if (result.status === 'rejected') {
+ deps.logger.error(
+ '[skill upload intent] Sweep item failed:',
+ getSafeErrorMetadata(result.reason),
+ );
+ }
+ }
+ }
+ async function tick() {
+ if (running) return;
+ running = true;
+ try {
+ await deps.runAsSystem(async () => {
+ try {
+ await deps.persistence.discoverUnjournaledDeletions(batchSize);
+ const tombstones = await deps.persistence.listSkillDeletionTombstones(batchSize);
+ const config = await deps.loadConfig();
+ if (config?.skillDeletion?.finalizationEnabled === true) {
+ const concurrency =
+ config?.fileConfig?.skills?.deletionCleanupConcurrency ??
+ DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY;
+ // Bound both active tombstones and their aggregate provider calls;
+ // each Skill also bounds its own file fanout in runSkillDeletion.
+ const limitTombstones = createConcurrencyLimiter(concurrency);
+ const limitBlob = createConcurrencyLimiter(concurrency);
+ await Promise.all(
+ tombstones.map((tombstone) =>
+ limitTombstones(async () => {
+ const skillId = tombstone.skillId;
+ try {
+ const progress = await deleteWithClaim(
+ skillId,
+ claimLeaseMs,
+ config,
+ limitBlob,
+ );
+ if (!progress.cleanupComplete) {
+ // A live upload is never timed out automatically. Operators must verify
+ // its writer is dead before explicitly abandoning the intent.
+ deps.logger.warn(
+ `[skill deletion] ${skillId} pending manual/retry cleanup: ${progress.failedSteps.join(', ')}`,
+ );
+ }
+ } catch (error) {
+ deps.logger.error(
+ `[skill deletion] Cleanup failed for ${skillId}:`,
+ getSafeErrorMetadata(error),
+ );
+ }
+ }),
+ ),
+ );
+ }
+ } finally {
+ // Active Skills can own abandoned upload intents with no deletion tombstone.
+ await sweepAbandonedIntents();
+ }
+ });
+ } catch (error) {
+ deps.logger.error('[skill deletion] Sweep failed:', getSafeErrorMetadata(error));
+ } finally {
+ running = false;
+ }
+ }
+ const interval = setInterval(tick, sweepIntervalMs);
+ interval.unref?.();
+ void tick();
+ return { stop: () => clearInterval(interval), runOnce: tick };
+ }
+
+ return { deleteSkillById, deleteSkillForLegacy, startSkillDeletionSweep };
+}
diff --git a/packages/data-schemas/src/methods/skill.ts b/packages/data-schemas/src/methods/skill.ts
index 8c25dda640..2d47244d5f 100644
--- a/packages/data-schemas/src/methods/skill.ts
+++ b/packages/data-schemas/src/methods/skill.ts
@@ -44,7 +44,11 @@ export type ValidationIssue = {
severity?: 'error' | 'warning';
};
-export type DeleteSkillCleanupStep = 'agent_allowlists' | 'skill_files' | 'permissions';
+export type DeleteSkillCleanupStep =
+ | 'agent_allowlists'
+ | 'skill_files'
+ | 'permissions'
+ | 'upload_intents';
export type DeleteSkillResult = {
/** Whether this call removed the Skill row. */