Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🧹 feat: Add Bounded Whole-Skill Cleanup Engine (#17042)
Dustin Healyc1815782738d
diff --git a/packages/api/src/skills/cleanup.spec.ts b/packages/api/src/skills/cleanup.spec.tsindex b974a241d7..f43d021c12 100644--- a/packages/api/src/skills/cleanup.spec.ts+++ b/packages/api/src/skills/cleanup.spec.ts@@ -30,6 +30,34 @@ describe('mergeDeleteSkillResults', () => {failedCleanupSteps: ['permissions'],});});++ it('does not lose a pending upload intent when a retry cannot confirm Skill absence', () => {+ expect(+ mergeDeleteSkillResults(incomplete('upload_intents'), {+ deleted: false,+ skillAbsent: false,+ cleanupComplete: false,+ failedCleanupSteps: [],+ }),+ ).toEqual({+ deleted: true,+ skillAbsent: true,+ cleanupComplete: false,+ failedCleanupSteps: ['upload_intents'],+ });+ });++ it('does not treat a late upload intent as settled by an earlier clean pass', () => {+ expect(+ mergeDeleteSkillResults(+ { deleted: true, skillAbsent: true, cleanupComplete: true, failedCleanupSteps: [] },+ incomplete('upload_intents'),+ ),+ ).toMatchObject({+ cleanupComplete: false,+ failedCleanupSteps: ['upload_intents'],+ });+ });});describe('deleteSkillWithRetry', () => {diff --git a/packages/api/src/skills/cleanup.ts b/packages/api/src/skills/cleanup.tsindex d0b6a35d33..77c8241da4 100644--- a/packages/api/src/skills/cleanup.ts+++ b/packages/api/src/skills/cleanup.ts@@ -1,21 +1,33 @@import type { DeleteSkillCleanupStep, DeleteSkillResult } from '@librechat/data-schemas';-const CLEANUP_STEPS: DeleteSkillCleanupStep[] = ['agent_allowlists', 'skill_files', 'permissions'];+const CLEANUP_STEPS: DeleteSkillCleanupStep[] = [+ 'agent_allowlists',+ 'skill_files',+ 'upload_intents',+ 'permissions',+];/**- * Cleanup is monotonic across idempotent deletion attempts. Once any attempt- * confirms a step succeeded, a later transient failure cannot make that work- * incomplete again. Only steps that failed in every confirming attempt remain- * unsettled.+ * Most cleanup steps are monotonic across idempotent attempts: one confirmed+ * success settles a transient failure. Upload intents are different because a+ * late writer can create one after an earlier pass, so the latest confirming+ * result decides whether that step is settled.*/export function mergeDeleteSkillResults(current: DeleteSkillResult,next: DeleteSkillResult,): DeleteSkillResult {const confirmingAttempts = [current, next].filter((result) => result.skillAbsent);- const failedCleanupSteps = CLEANUP_STEPS.filter((step) =>- confirmingAttempts.every((result) => result.failedCleanupSteps.includes(step)),- );+ const failedCleanupSteps = CLEANUP_STEPS.filter((step) => {+ if (step === 'upload_intents') {+ // A late writer can introduce a new intent after an earlier clean pass.+ // Only a newer confirming pass may settle this non-monotonic step.+ return next.skillAbsent+ ? next.failedCleanupSteps.includes(step)+ : current.failedCleanupSteps.includes(step);+ }+ return confirmingAttempts.every((result) => result.failedCleanupSteps.includes(step));+ });const skillAbsent = confirmingAttempts.length > 0;return {diff --git a/packages/api/src/skills/deletion.spec.ts b/packages/api/src/skills/deletion.spec.tsnew file mode 100644index 0000000000..f607de0b72--- /dev/null+++ b/packages/api/src/skills/deletion.spec.ts@@ -0,0 +1,300 @@+import type { SkillDeletionRecord, SkillUploadIntentRecord } from '@librechat/data-schemas';+import type { SkillDeletionDeps } from './deletion';+import { runSkillDeletion, SkillDeletionClaimLostError } from './deletion';++const skillId = '507f1f77bcf86cd799439011';+const author = '507f1f77bcf86cd799439012';+const journal = {+ skillId,+} satisfies SkillDeletionRecord;++function harness() {+ const files = [+ {+ file_id: 'file-1',+ filepath: '/uploads/owner/file-1',+ source: 'local',+ author,+ },+ ] satisfies Awaited<ReturnType<SkillDeletionDeps['listSkillFiles']>>;+ const intents: SkillUploadIntentRecord[] = [];+ const deps: SkillDeletionDeps = {+ beginSkillDeletion: jest.fn(async () => journal),+ getSkillDeletion: jest.fn(async () => journal),+ listSkillFiles: jest.fn(async () => files),+ listSkillUploadIntents: jest.fn(async () => intents),+ deleteBlob: jest.fn(async () => undefined),+ deleteReservedBlob: jest.fn(async () => undefined),+ deleteSkillFileAfterBlob: jest.fn(async (_id, fileId) => {+ const index = files.findIndex((file) => file.file_id === fileId);+ if (index < 0) return false;+ files.splice(index, 1);+ return true;+ }),+ finalizeSkillDeletion: jest.fn(async () => ({+ deleted: true,+ skillAbsent: true,+ cleanupComplete: true,+ failedCleanupSteps: [],+ })),+ completeSkillDeletion: jest.fn(async () => true),+ reopenSkillDeletion: jest.fn(async () => undefined),+ markSkillUploadIntentCleaned: jest.fn(async (fileId) => {+ const intent = intents.find((row) => row.fileId === fileId);+ if (intent) intent.cleanedAt = new Date();+ }),+ };+ return { files, intents, deps };+}++it('keeps the file row when provider deletion fails and succeeds on retry', async () => {+ const h = harness();+ const deleteBlob = h.deps.deleteBlob as jest.Mock;+ deleteBlob.mockRejectedValueOnce(new Error('storage unavailable'));+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({+ cleanupComplete: false,+ failedSteps: ['skill_files'],+ });+ expect(h.files).toHaveLength(1);+ expect(h.deps.finalizeSkillDeletion).not.toHaveBeenCalled();+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({+ cleanupComplete: true,+ });+ expect(h.files).toHaveLength(0);+ expect(h.deps.completeSkillDeletion).toHaveBeenCalledTimes(1);+});++it('does not report completion when the claim transfers after its final refresh', async () => {+ const h = harness();+ h.files.length = 0;+ (h.deps.completeSkillDeletion as jest.Mock).mockResolvedValueOnce(false);+ await expect(runSkillDeletion(skillId, h.deps)).rejects.toBeInstanceOf(+ SkillDeletionClaimLostError,+ );+ expect(h.deps.completeSkillDeletion).toHaveBeenCalledWith(skillId);+});++it('preserves idempotent absence and per-call deletion semantics', async () => {+ const h = harness();+ (h.deps.beginSkillDeletion as jest.Mock).mockResolvedValueOnce(null);+ (h.deps.getSkillDeletion as jest.Mock).mockResolvedValueOnce(null);+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({+ found: false,+ deleted: false,+ skillAbsent: true,+ cleanupComplete: true,+ });++ h.files.length = 0;+ (h.deps.finalizeSkillDeletion as jest.Mock).mockResolvedValueOnce({+ deleted: false,+ skillAbsent: true,+ cleanupComplete: true,+ failedCleanupSteps: [],+ });+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({+ found: true,+ deleted: false,+ skillAbsent: true,+ cleanupComplete: true,+ });+});++it.each(['uploading', 'committing'] as const)(+ 'leaves an in-flight %s writer pending without deleting its blob',+ async (state) => {+ const h = harness();+ h.intents.push({+ skillId,+ author,+ fileId: 'file-1',+ source: 'local',+ storageKey: '/uploads/owner/file-1',+ state,+ });+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({+ cleanupComplete: false,+ failedSteps: ['upload_intents'],+ });+ expect(h.deps.deleteBlob).not.toHaveBeenCalled();+ expect(h.deps.finalizeSkillDeletion).not.toHaveBeenCalled();+ },+);++it('cleans a committed historical blob after its SkillFile row was replaced', async () => {+ const h = harness();+ h.files.length = 0;+ h.intents.push({+ skillId,+ author,+ fileId: 'old-file',+ source: 'local',+ storageKey: '/uploads/owner/old-file',+ state: 'committed',+ });+ const first = await runSkillDeletion(skillId.toString(), h.deps);+ expect(first.cleanupComplete).toBe(true);+ expect(h.deps.deleteReservedBlob).toHaveBeenCalledTimes(1);+ expect(h.intents[0].cleanedAt).toBeDefined();+ await runSkillDeletion(skillId.toString(), h.deps);+ expect(h.deps.deleteReservedBlob).toHaveBeenCalledTimes(1);+});++it('deletes a current file blob only once when its committed intent points at the same upload', async () => {+ const h = harness();+ h.intents.push({+ skillId,+ author,+ fileId: 'file-1',+ source: 'local',+ storageKey: '/uploads/owner/file-1',+ state: 'committed',+ });+ await expect(runSkillDeletion(skillId.toString(), h.deps)).resolves.toMatchObject({+ cleanupComplete: true,+ });+ expect(h.deps.deleteBlob).toHaveBeenCalledTimes(1);+ expect(h.deps.deleteReservedBlob).not.toHaveBeenCalled();+ expect(h.intents[0].cleanedAt).toBeDefined();+});++it('resweeps abandoned intents even after an earlier clean pass', async () => {+ const h = harness();+ h.files.length = 0;+ h.intents.push({+ skillId,+ author,+ fileId: 'zombie',+ source: 'local',+ storageKey: '/uploads/owner/zombie',+ state: 'abandoned',+ });+ await runSkillDeletion(skillId.toString(), h.deps);+ await runSkillDeletion(skillId.toString(), h.deps);+ expect(h.deps.deleteReservedBlob).toHaveBeenCalledTimes(2);+});++it('cleans intent blobs with bounded concurrency', async () => {+ const h = harness();+ h.files.length = 0;+ for (let index = 0; index < 4; index++) {+ h.intents.push({+ skillId,+ author,+ fileId: `orphan-${index}`,+ source: 'local',+ storageKey: `/uploads/owner/orphan-${index}`,+ state: 'abandoned',+ });+ }+ let active = 0;+ let peak = 0;+ h.deps.deleteReservedBlob = jest.fn(async () => {+ active++;+ peak = Math.max(peak, active);+ await new Promise((resolve) => setImmediate(resolve));+ active--;+ });++ await runSkillDeletion(skillId, h.deps, 2);+ expect(h.deps.deleteReservedBlob).toHaveBeenCalledTimes(4);+ expect(peak).toBe(2);+});++it('does not reopen or finalize an already-completed tombstone with no late work', async () => {+ const h = harness();+ h.files.length = 0;+ (h.deps.beginSkillDeletion as jest.Mock).mockResolvedValue({+ skillId,+ completedAt: new Date(),+ });+ await expect(runSkillDeletion(skillId, h.deps)).resolves.toMatchObject({+ cleanupComplete: true,+ skillAbsent: true,+ });+ expect(h.deps.reopenSkillDeletion).not.toHaveBeenCalled();+ expect(h.deps.finalizeSkillDeletion).not.toHaveBeenCalled();+ expect(h.deps.completeSkillDeletion).not.toHaveBeenCalled();+});++it('cold-audits an abandoned intent without rerunning completed dependent cleanup', async () => {+ const h = harness();+ h.files.length = 0;+ h.intents.push({+ skillId,+ author,+ fileId: 'late-write',+ source: 'local',+ storageKey: '/uploads/late-write',+ state: 'abandoned',+ cleanedAt: new Date(),+ });+ (h.deps.beginSkillDeletion as jest.Mock).mockResolvedValue({+ skillId,+ completedAt: new Date(),+ });+ await expect(runSkillDeletion(skillId, h.deps)).resolves.toMatchObject({+ cleanupComplete: true,+ });+ expect(h.deps.deleteReservedBlob).toHaveBeenCalledTimes(1);+ expect(h.deps.reopenSkillDeletion).not.toHaveBeenCalled();+ expect(h.deps.finalizeSkillDeletion).not.toHaveBeenCalled();+});++it('reopens completed deletion only when a late file needs cleanup', async () => {+ const h = harness();+ (h.deps.beginSkillDeletion as jest.Mock).mockResolvedValue({+ skillId,+ completedAt: new Date(),+ });+ await expect(runSkillDeletion(skillId, h.deps)).resolves.toMatchObject({+ cleanupComplete: true,+ });+ expect(h.deps.reopenSkillDeletion).toHaveBeenCalledWith(skillId);+ expect(h.deps.deleteBlob).toHaveBeenCalledTimes(1);+ expect(h.deps.finalizeSkillDeletion).toHaveBeenCalledTimes(1);+});++it('does not mutate a file row after losing the claim during its provider delete', async () => {+ const h = harness();+ let releaseBlob: (() => void) | undefined;+ let startedBlob: (() => void) | undefined;+ const started = new Promise<void>((resolve) => {+ startedBlob = resolve;+ });+ (h.deps.deleteBlob as jest.Mock).mockImplementation(+ () =>+ new Promise<void>((resolve) => {+ releaseBlob = resolve;+ startedBlob?.();+ }),+ );+ let lost = false;+ const assertClaim = jest.fn(async () => {+ if (lost) throw new SkillDeletionClaimLostError();+ });+ const deletion = runSkillDeletion(skillId, h.deps, 1, assertClaim);+ await started;+ lost = true;+ releaseBlob?.();+ await expect(deletion).rejects.toBeInstanceOf(SkillDeletionClaimLostError);+ expect(h.deps.deleteSkillFileAfterBlob).not.toHaveBeenCalled();+ expect(h.deps.finalizeSkillDeletion).not.toHaveBeenCalled();+});++it('bounds independent file cleanup while deleting each row after its blob', async () => {+ const h = harness();+ h.files.push({ ...h.files[0], file_id: 'file-2' }, { ...h.files[0], file_id: 'file-3' });+ let active = 0;+ let peak = 0;+ (h.deps.deleteBlob as jest.Mock).mockImplementation(async () => {+ active++;+ peak = Math.max(peak, active);+ await new Promise((resolve) => setImmediate(resolve));+ active--;+ });+ await runSkillDeletion(skillId, h.deps, 2);+ expect(peak).toBe(2);+ expect(h.deps.deleteSkillFileAfterBlob).toHaveBeenCalledTimes(3);+ expect(h.files).toHaveLength(0);+});diff --git a/packages/api/src/skills/deletion.ts b/packages/api/src/skills/deletion.tsnew file mode 100644index 0000000000..cf237a2a69--- /dev/null+++ b/packages/api/src/skills/deletion.ts@@ -0,0 +1,198 @@+import { DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY } from 'librechat-data-provider';+import type {+ DeleteSkillCleanupStep,+ DeleteSkillResult,+ SkillDeletionFileRecord,+ SkillDeletionRecord,+ SkillUploadIntentRecord,+} from '@librechat/data-schemas';+import { createConcurrencyLimiter } from '~/utils/promise';++export interface SkillDeletionDeps {+ beginSkillDeletion: (id: string) => Promise<SkillDeletionRecord | null>;+ getSkillDeletion: (id: string) => Promise<SkillDeletionRecord | null>;+ listSkillFiles: (id: string) => Promise<SkillDeletionFileRecord[]>;+ listSkillUploadIntents: (id: string) => Promise<SkillUploadIntentRecord[]>;+ deleteBlob: (file: SkillDeletionFileRecord) => Promise<void>;+ deleteReservedBlob: (intent: SkillUploadIntentRecord) => Promise<void>;+ deleteSkillFileAfterBlob: (skillId: string, fileId: string) => Promise<boolean>;+ finalizeSkillDeletion: (id: string) => Promise<DeleteSkillResult>;+ completeSkillDeletion: (id: string) => Promise<boolean>;+ reopenSkillDeletion: (id: string) => Promise<void>;+ markSkillUploadIntentCleaned: (fileId: string) => Promise<void>;+}++export type SkillDeletionProgress = {+ found: boolean;+ deleted: boolean;+ skillAbsent: boolean;+ cleanupComplete: boolean;+ failedSteps: DeleteSkillCleanupStep[];+};++export class SkillDeletionClaimLostError extends Error {+ constructor() {+ super('Skill deletion claim was lost');+ this.name = 'SkillDeletionClaimLostError';+ }+}++/**+ * Every externally stored object has a persistent Mongo reference until its+ * provider and RAG cleanup succeeds. Failures remain retryable after restart.+ */+export async function runSkillDeletion(+ id: string,+ deps: SkillDeletionDeps,+ fileCleanupConcurrency: number = DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY,+ assertClaim: () => Promise<void> = async () => undefined,+ loadedJournal?: SkillDeletionRecord,+): Promise<SkillDeletionProgress> {+ await assertClaim();+ const journal =+ loadedJournal ?? (await deps.beginSkillDeletion(id)) ?? (await deps.getSkillDeletion(id));+ if (!journal) {+ return {+ found: false,+ deleted: false,+ skillAbsent: true,+ cleanupComplete: true,+ failedSteps: [],+ };+ }+ const failedSteps: DeleteSkillCleanupStep[] = [];+ const [intents, files] = await Promise.all([+ deps.listSkillUploadIntents(journal.skillId),+ deps.listSkillFiles(id),+ ]);+ const requiresFullCleanup =+ files.length > 0 ||+ intents.some(+ (intent) =>+ intent.state === 'uploading' ||+ intent.state === 'committing' ||+ (intent.state === 'committed' && !intent.cleanedAt),+ );+ if (journal.completedAt && requiresFullCleanup) {+ await assertClaim();+ await deps.reopenSkillDeletion(journal.skillId);+ }+ // A writer may still finish uploading or committing after this scan. An+ // operator can inspect and recover an intent with `recover-skill-upload`+ // after draining writers; a timer must not infer that a slow upload is dead.+ if (intents.some((intent) => intent.state === 'uploading' || intent.state === 'committing')) {+ return {+ found: true,+ deleted: false,+ skillAbsent: journal.completedAt != null,+ cleanupComplete: false,+ failedSteps: ['upload_intents'],+ };+ }+ const currentFileIds = new Set(files.map((file) => file.file_id));+ const cleanedFileIds = new Set<string>();+ const limit = createConcurrencyLimiter(fileCleanupConcurrency);+ const fileResults = await Promise.allSettled(+ files.map((file) =>+ limit(async () => {+ await assertClaim();+ try {+ await deps.deleteBlob(file);+ } catch {+ failedSteps.push('skill_files');+ return;+ }+ await assertClaim();+ try {+ await deps.deleteSkillFileAfterBlob(id, file.file_id);+ cleanedFileIds.add(file.file_id);+ } catch {+ failedSteps.push('skill_files');+ }+ }),+ ),+ );+ const rejectedFile = fileResults.find((result) => result.status === 'rejected');+ if (rejectedFile?.status === 'rejected') throw rejectedFile.reason;++ const intentResults = await Promise.allSettled(+ intents.map((intent) =>+ limit(async () => {+ if (intent.state === 'committed') {+ if (intent.cleanedAt) return;+ if (currentFileIds.has(intent.fileId) && !cleanedFileIds.has(intent.fileId)) {+ // Keep the intent with its current row for the next retry.+ return;+ }+ await assertClaim();+ try {+ if (!cleanedFileIds.has(intent.fileId)) {+ await deps.deleteReservedBlob(intent);+ }+ } catch {+ failedSteps.push('upload_intents');+ return;+ }+ await assertClaim();+ try {+ await deps.markSkillUploadIntentCleaned(intent.fileId);+ } catch {+ failedSteps.push('upload_intents');+ }+ return;+ }+ await assertClaim();+ try {+ await deps.deleteReservedBlob(intent);+ // Keep abandoned intents as tombstones: a zombie upload can finish+ // after this cleanup and must be discovered on the next sweep.+ } catch {+ failedSteps.push('upload_intents');+ }+ }),+ ),+ );+ const rejectedIntent = intentResults.find((result) => result.status === 'rejected');+ if (rejectedIntent?.status === 'rejected') throw rejectedIntent.reason;++ if (failedSteps.length > 0) {+ return {+ found: true,+ deleted: false,+ skillAbsent: journal.completedAt != null,+ cleanupComplete: false,+ failedSteps,+ };+ }+ if (journal.completedAt && !requiresFullCleanup) {+ return {+ found: true,+ deleted: true,+ skillAbsent: true,+ cleanupComplete: true,+ failedSteps: [],+ };+ }+ await assertClaim();+ const finalized = await deps.finalizeSkillDeletion(id);+ if (!finalized.cleanupComplete) {+ return {+ found: true,+ deleted: finalized.deleted,+ skillAbsent: finalized.skillAbsent,+ cleanupComplete: false,+ failedSteps: finalized.failedCleanupSteps,+ };+ }+ await assertClaim();+ if (!(await deps.completeSkillDeletion(journal.skillId))) {+ throw new SkillDeletionClaimLostError();+ }+ return {+ found: true,+ deleted: finalized.deleted,+ skillAbsent: finalized.skillAbsent,+ cleanupComplete: true,+ failedSteps: [],+ };+}diff --git a/packages/api/src/skills/deletionService.spec.ts b/packages/api/src/skills/deletionService.spec.tsnew file mode 100644index 0000000000..3c271df4a9--- /dev/null+++ b/packages/api/src/skills/deletionService.spec.ts@@ -0,0 +1,548 @@+import type { AppConfig, SkillUploadIntentRecord } from '@librechat/data-schemas';+import { createSkillDeletionService } from './deletionService';++it('records failed cleanup attempts before releasing the claim', async () => {+ const skillId = '507f1f77bcf86cd799439011';+ const journal = { skillId };+ const order: string[] = [];+ let released: () => void = () => undefined;+ const releaseObserved = new Promise<void>((resolve) => {+ released = resolve;+ });+ const persistence = {+ discoverUnjournaledDeletions: jest.fn(async () => 0),+ listSkillDeletionTombstones: jest.fn(async () => [journal]),+ claimSkillDeletion: jest.fn(async () => true),+ refreshSkillDeletionClaim: jest.fn(async () => true),+ noteSkillDeletionAttempt: jest.fn(async () => {+ order.push('attempt');+ }),+ releaseSkillDeletionClaim: jest.fn(async (): Promise<void> => {+ order.push('release');+ released();+ }),+ beginSkillDeletion: jest.fn(async () => journal),+ getSkillDeletion: jest.fn(async (): Promise<typeof journal | null> => journal),+ listSkillFiles: jest.fn(async () => {+ throw new Error('storage unavailable');+ }),+ listSkillUploadIntents: jest.fn(async () => []),+ deleteSkillFileAfterBlob: jest.fn(async () => true),+ finalizeSkillDeletion: jest.fn(async () => ({+ deleted: true,+ skillAbsent: true,+ cleanupComplete: true,+ failedCleanupSteps: [],+ })),+ completeSkillDeletion: jest.fn(async () => true),+ reopenSkillDeletion: jest.fn(async () => undefined),+ markSkillUploadIntentCleaned: jest.fn(async () => undefined),+ listAbandonedSkillUploadIntents: jest.fn(async (): Promise<SkillUploadIntentRecord[]> => []),+ claimAbandonedSkillUploadIntent: jest.fn(async () => true),+ refreshAbandonedSkillUploadIntentClaim: jest.fn(async () => true),+ noteAbandonedSkillUploadIntentAttempt: jest.fn(async () => undefined),+ releaseAbandonedSkillUploadIntentClaim: jest.fn(async () => undefined),+ scheduleSkillDeletionRecheck: jest.fn(async () => true),+ scheduleAbandonedSkillUploadIntentRecheck: jest.fn(async () => true),+ skillFileExistsForIntent: jest.fn(async () => false),+ };+ const service = createSkillDeletionService({+ persistence,+ loadConfig: jest.fn(+ async () => ({ skillDeletion: { finalizationEnabled: true } }) as unknown as AppConfig,+ ),+ getStrategyFunctions: jest.fn(() => ({})),+ runAsSystem: async (callback) => callback(),+ logger: { error: jest.fn(), warn: jest.fn() },+ });++ const sweep = service.startSkillDeletionSweep({ sweepIntervalMs: 60_000, claimLeaseMs: 120_000 });+ await releaseObserved;+ sweep.stop();+ expect(order).toEqual(['attempt', 'release']);+ expect(persistence.beginSkillDeletion).toHaveBeenCalledTimes(1);+ expect(persistence.getSkillDeletion).not.toHaveBeenCalled();+ expect(persistence.noteSkillDeletionAttempt).toHaveBeenCalledWith(skillId);+});++it.each([false, true])(+ 'reconciles an abandoned intent on an active Skill while preserving a live file row: row=%s',+ async (rowExists) => {+ const skillId = '507f1f77bcf86cd799439011';+ const author = '507f1f77bcf86cd799439012';+ const intent = {+ skillId,+ fileId: 'abandoned-file',+ source: 'local',+ storageKey: '/uploads/abandoned-file',+ author,+ state: 'abandoned' as const,+ };+ let released: () => void = () => undefined;+ const releaseObserved = new Promise<void>((resolve) => {+ released = resolve;+ });+ const deleteFile = jest.fn(async () => undefined);+ const persistence = {+ discoverUnjournaledDeletions: jest.fn(async () => 0),+ listSkillDeletionTombstones: jest.fn(async () => []),+ claimSkillDeletion: jest.fn(async () => true),+ refreshSkillDeletionClaim: jest.fn(async () => true),+ noteSkillDeletionAttempt: jest.fn(async () => undefined),+ releaseSkillDeletionClaim: jest.fn(async () => undefined),+ beginSkillDeletion: jest.fn(async () => null),+ getSkillDeletion: jest.fn(async () => null),+ listSkillFiles: jest.fn(async () => []),+ listSkillUploadIntents: jest.fn(async () => []),+ deleteSkillFileAfterBlob: jest.fn(async () => true),+ finalizeSkillDeletion: jest.fn(async () => ({+ deleted: false,+ skillAbsent: true,+ cleanupComplete: true,+ failedCleanupSteps: [],+ })),+ completeSkillDeletion: jest.fn(async () => true),+ reopenSkillDeletion: jest.fn(async () => undefined),+ markSkillUploadIntentCleaned: jest.fn(async () => undefined),+ listAbandonedSkillUploadIntents: jest.fn(async () => [intent]),+ claimAbandonedSkillUploadIntent: jest.fn(async () => true),+ refreshAbandonedSkillUploadIntentClaim: jest.fn(async () => true),+ noteAbandonedSkillUploadIntentAttempt: jest.fn(async () => undefined),+ releaseAbandonedSkillUploadIntentClaim: jest.fn(async () => released()),+ scheduleSkillDeletionRecheck: jest.fn(async () => true),+ scheduleAbandonedSkillUploadIntentRecheck: jest.fn(async () => true),+ skillFileExistsForIntent: jest.fn(async () => rowExists),+ };+ const service = createSkillDeletionService({+ persistence,+ loadConfig: jest.fn(async () => undefined),+ getStrategyFunctions: jest.fn(() => ({ deleteFile })),+ runAsSystem: async (callback) => callback(),+ logger: { error: jest.fn(), warn: jest.fn() },+ });++ const sweep = service.startSkillDeletionSweep();+ await releaseObserved;+ sweep.stop();+ expect(deleteFile).toHaveBeenCalledTimes(rowExists ? 0 : 1);+ expect(persistence.markSkillUploadIntentCleaned).toHaveBeenCalledTimes(rowExists ? 0 : 1);+ expect(persistence.scheduleAbandonedSkillUploadIntentRecheck).toHaveBeenCalledWith(+ intent.fileId,+ expect.any(String),+ 3_600_000,+ );+ expect(persistence.noteAbandonedSkillUploadIntentAttempt).toHaveBeenCalledWith(intent.fileId);+ },+);++function claimedServiceHarness() {+ const skillId = '507f1f77bcf86cd799439011';+ const journal = { skillId, completedAt: undefined as Date | undefined };+ const files = [+ {+ file_id: 'file-1',+ filepath: '/uploads/file-1',+ source: 'local',+ author: '507f1f77bcf86cd799439012',+ },+ ];+ let held = false;+ const persistence = {+ discoverUnjournaledDeletions: jest.fn(async () => 0),+ listSkillDeletionTombstones: jest.fn(async () => [journal]),+ claimSkillDeletion: jest.fn(async () => {+ if (held) return false;+ held = true;+ return true;+ }),+ refreshSkillDeletionClaim: jest.fn(async () => held),+ noteSkillDeletionAttempt: jest.fn(async () => undefined),+ releaseSkillDeletionClaim: jest.fn(async (): Promise<void> => {+ held = false;+ }),+ scheduleSkillDeletionRecheck: jest.fn(async () => true),+ beginSkillDeletion: jest.fn(async () => journal),+ getSkillDeletion: jest.fn(async (): Promise<typeof journal | null> => journal),+ listSkillFiles: jest.fn(async () => files),+ listSkillUploadIntents: jest.fn(async () => []),+ deleteSkillFileAfterBlob: jest.fn(async () => {+ files.length = 0;+ return true;+ }),+ finalizeSkillDeletion: jest.fn(async () => ({+ deleted: true,+ skillAbsent: true,+ cleanupComplete: true,+ failedCleanupSteps: [],+ })),+ completeSkillDeletion: jest.fn(async () => {+ journal.completedAt = new Date();+ return true;+ }),+ reopenSkillDeletion: jest.fn(async () => undefined),+ markSkillUploadIntentCleaned: jest.fn(async () => undefined),+ listAbandonedSkillUploadIntents: jest.fn(async (): Promise<SkillUploadIntentRecord[]> => []),+ claimAbandonedSkillUploadIntent: jest.fn(async () => true),+ refreshAbandonedSkillUploadIntentClaim: jest.fn(async () => true),+ noteAbandonedSkillUploadIntentAttempt: jest.fn(async () => undefined),+ releaseAbandonedSkillUploadIntentClaim: jest.fn(async () => undefined),+ scheduleAbandonedSkillUploadIntentRecheck: jest.fn(async () => true),+ skillFileExistsForIntent: jest.fn(async () => false),+ };+ const deleteFile = jest.fn(async (): Promise<void> => undefined);+ const loadConfig = jest.fn(+ async (): Promise<AppConfig | undefined> =>+ ({ skillDeletion: { finalizationEnabled: true } }) as unknown as AppConfig,+ );+ const logger = { error: jest.fn(), warn: jest.fn() };+ const service = createSkillDeletionService({+ persistence,+ loadConfig,+ getStrategyFunctions: jest.fn(() => ({ deleteFile })),+ runAsSystem: async (callback) => callback(),+ logger,+ });+ return { skillId, persistence, deleteFile, loadConfig, logger, service };+}++it('marks deletion but cannot finalize until the writer-drain gate is enabled', async () => {+ const h = claimedServiceHarness();+ h.loadConfig.mockResolvedValue({+ skillDeletion: { finalizationEnabled: false },+ } as unknown as AppConfig);++ const pending = await h.service.deleteSkillById(h.skillId);+ expect(pending).toMatchObject({ found: true, deleted: false, cleanupComplete: false });+ expect(h.persistence.beginSkillDeletion).toHaveBeenCalledWith(h.skillId);+ expect(h.persistence.claimSkillDeletion).not.toHaveBeenCalled();+ expect(h.persistence.finalizeSkillDeletion).not.toHaveBeenCalled();++ h.loadConfig.mockResolvedValue({+ skillDeletion: { finalizationEnabled: true },+ } as unknown as AppConfig);+ const completed = await h.service.deleteSkillById(h.skillId);+ expect(completed.cleanupComplete).toBe(true);+ expect(h.persistence.finalizeSkillDeletion).toHaveBeenCalledTimes(1);+});++it('returns pending when the claim transfers between its final check and completion write', async () => {+ const h = claimedServiceHarness();+ h.persistence.completeSkillDeletion.mockResolvedValueOnce(false);++ const progress = await h.service.deleteSkillById(h.skillId);+ expect(progress).toMatchObject({ found: true, deleted: false, cleanupComplete: false });+ expect(h.persistence.completeSkillDeletion).toHaveBeenCalledWith(h.skillId, expect.any(String));+ expect(h.persistence.releaseSkillDeletionClaim).toHaveBeenCalledTimes(1);+});++it('processes independent deletion tombstones with bounded concurrency', async () => {+ const h = claimedServiceHarness();+ h.persistence.listSkillDeletionTombstones.mockResolvedValue(+ ['507f1f77bcf86cd799439011', '507f1f77bcf86cd799439012', '507f1f77bcf86cd799439013'].map(+ (skillId) => ({ skillId, completedAt: undefined }),+ ),+ );+ h.persistence.claimSkillDeletion.mockResolvedValue(true);+ h.persistence.refreshSkillDeletionClaim.mockResolvedValue(true);+ h.persistence.listSkillFiles.mockImplementation(async () => [+ {+ file_id: 'file-1',+ filepath: '/uploads/file-1',+ source: 'local',+ author: '507f1f77bcf86cd799439012',+ },+ {+ file_id: 'file-2',+ filepath: '/uploads/file-2',+ source: 'local',+ author: '507f1f77bcf86cd799439012',+ },+ ]);+ h.persistence.deleteSkillFileAfterBlob.mockResolvedValue(true);+ h.loadConfig.mockResolvedValue({+ skillDeletion: { finalizationEnabled: true },+ fileConfig: { skills: { deletionCleanupConcurrency: 2 } },+ } as unknown as AppConfig);+ let active = 0;+ let peak = 0;+ let allCompleted!: () => void;+ const done = new Promise<void>((resolve) => {+ allCompleted = resolve;+ });+ let completed = 0;+ h.persistence.releaseSkillDeletionClaim.mockImplementation(async () => {+ completed++;+ if (completed === 3) allCompleted();+ });+ h.deleteFile.mockImplementation(async () => {+ active++;+ peak = Math.max(peak, active);+ await new Promise<void>((resolve) => setTimeout(resolve, 10));+ active--;+ });++ const sweep = h.service.startSkillDeletionSweep();+ try {+ await done;+ expect(peak).toBe(2);+ expect(h.deleteFile).toHaveBeenCalledTimes(6);+ expect(h.persistence.finalizeSkillDeletion).toHaveBeenCalledTimes(3);+ expect(h.loadConfig).toHaveBeenCalledTimes(1);+ } finally {+ sweep.stop();+ }+});++it('uses the configured claim lease for direct deletion', async () => {+ const h = claimedServiceHarness();+ h.loadConfig.mockResolvedValue({+ skillDeletion: { finalizationEnabled: true, claimLeaseMs: 9_000 },+ } as unknown as AppConfig);+ await h.service.deleteSkillById(h.skillId);+ expect(h.persistence.claimSkillDeletion).toHaveBeenCalledWith(+ h.skillId,+ expect.any(String),+ 9_000,+ );+ expect(h.loadConfig).toHaveBeenCalledTimes(1);+ expect(h.persistence.beginSkillDeletion).toHaveBeenCalledTimes(1);+ expect(h.persistence.getSkillDeletion).not.toHaveBeenCalled();+});++it('does not log provider metadata when abandoned-intent cleanup fails', async () => {+ const h = claimedServiceHarness();+ const fileId = 'abandoned-file';+ h.persistence.listSkillDeletionTombstones.mockResolvedValue([]);+ h.persistence.listAbandonedSkillUploadIntents.mockResolvedValue([+ {+ skillId: h.skillId,+ fileId,+ source: 'local',+ storageKey: '/uploads/abandoned-file',+ author: '507f1f77bcf86cd799439012',+ state: 'abandoned',+ },+ ]);+ h.persistence.getSkillDeletion.mockResolvedValue(null);+ h.deleteFile.mockRejectedValue(+ Object.assign(new Error('signed URL: secret-token'), {+ status: 403,+ request: { url: 'https://storage.example/object?signature=secret-token' },+ response: { headers: { authorization: 'secret-token' } },+ }),+ );+ let released!: () => void;+ const done = new Promise<void>((resolve) => {+ released = resolve;+ });+ h.persistence.releaseAbandonedSkillUploadIntentClaim.mockImplementation(async () => {+ released();+ return undefined;+ });++ const sweep = h.service.startSkillDeletionSweep();+ await done;+ sweep.stop();++ expect(h.logger.error).toHaveBeenCalledWith(+ `[skill upload intent] Cleanup failed for ${fileId}:`,+ { type: 'Error', status: 403 },+ );+ expect(JSON.stringify(h.logger.error.mock.calls)).not.toContain('secret-token');+});++it('bounds independent abandoned-intent cleanup without serializing the batch', async () => {+ const h = claimedServiceHarness();+ h.persistence.listSkillDeletionTombstones.mockResolvedValue([]);+ h.persistence.listAbandonedSkillUploadIntents.mockResolvedValue(+ ['file-1', 'file-2', 'file-3'].map((fileId) => ({+ skillId: h.skillId,+ fileId,+ source: 'local',+ storageKey: `/uploads/${fileId}`,+ author: '507f1f77bcf86cd799439012',+ state: 'abandoned' as const,+ })),+ );+ h.persistence.getSkillDeletion.mockResolvedValue(null);+ h.loadConfig.mockResolvedValue({+ fileConfig: { skills: { deletionCleanupConcurrency: 2 } },+ } as AppConfig);+ let active = 0;+ let peak = 0;+ let started = 0;+ let completed = 0;+ const releases: Array<() => void> = [];+ let twoStarted!: () => void;+ let threeStarted!: () => void;+ let allReleased!: () => void;+ const firstPair = new Promise<void>((resolve) => {+ twoStarted = resolve;+ });+ const third = new Promise<void>((resolve) => {+ threeStarted = resolve;+ });+ const done = new Promise<void>((resolve) => {+ allReleased = resolve;+ });+ h.deleteFile.mockImplementation(async () => {+ active++;+ peak = Math.max(peak, active);+ started++;+ if (started === 2) twoStarted();+ if (started === 3) threeStarted();+ await new Promise<void>((resolve) => releases.push(resolve));+ active--;+ });+ h.persistence.releaseAbandonedSkillUploadIntentClaim.mockImplementation(async () => {+ completed++;+ if (completed === 3) allReleased();+ return undefined;+ });++ const sweep = h.service.startSkillDeletionSweep();+ try {+ await firstPair;+ expect(h.deleteFile).toHaveBeenCalledTimes(2);+ releases[0]();+ await third;+ expect(peak).toBe(2);+ for (const release of releases.slice(1)) release();+ await done;+ expect(h.persistence.markSkillUploadIntentCleaned).toHaveBeenCalledTimes(3);+ expect(h.loadConfig).toHaveBeenCalledTimes(2);+ } finally {+ sweep.stop();+ }+});++it('defers tombstone-owned abandoned intents without deleting their blobs again', async () => {+ const h = claimedServiceHarness();+ const intent = {+ skillId: h.skillId,+ fileId: 'tombstone-owned',+ source: 'local',+ storageKey: '/uploads/tombstone-owned',+ author: '507f1f77bcf86cd799439012',+ state: 'abandoned' as const,+ };+ h.persistence.listSkillDeletionTombstones.mockResolvedValue([]);+ h.persistence.listAbandonedSkillUploadIntents.mockResolvedValue([intent]);+ let released: (() => void) | undefined;+ const done = new Promise<void>((resolve) => {+ released = resolve;+ });+ h.persistence.releaseAbandonedSkillUploadIntentClaim.mockImplementation(async () => {+ released?.();+ });+ const sweep = h.service.startSkillDeletionSweep();+ await done;+ sweep.stop();+ expect(h.persistence.scheduleAbandonedSkillUploadIntentRecheck).toHaveBeenCalledWith(+ intent.fileId,+ expect.any(String),+ 3_600_000,+ );+ expect(h.deleteFile).not.toHaveBeenCalled();+});++it('returns pending to a direct caller while the sweep owns the same Skill claim', async () => {+ const h = claimedServiceHarness();+ let releasedClaim: (() => void) | undefined;+ const released = new Promise<void>((resolve) => {+ releasedClaim = resolve;+ });+ h.persistence.releaseSkillDeletionClaim.mockImplementation(async () => {+ releasedClaim?.();+ });+ let releaseBlob: (() => void) | undefined;+ let startedBlob: (() => void) | undefined;+ const started = new Promise<void>((resolve) => {+ startedBlob = resolve;+ });+ h.deleteFile.mockImplementation(+ () =>+ new Promise<void>((resolve) => {+ releaseBlob = resolve;+ startedBlob?.();+ }),+ );+ const sweep = h.service.startSkillDeletionSweep();+ await started;+ const pending = await h.service.deleteSkillById(h.skillId);+ expect(pending).toMatchObject({ found: true, cleanupComplete: false });+ expect(h.deleteFile).toHaveBeenCalledTimes(1);+ releaseBlob?.();+ await released;+ sweep.stop();+ expect(h.persistence.finalizeSkillDeletion).toHaveBeenCalledTimes(1);+});++it('treats a deferred completed tombstone as an idempotent direct deletion', async () => {+ const h = claimedServiceHarness();+ const completed = { skillId: h.skillId, completedAt: new Date() };+ h.persistence.beginSkillDeletion.mockResolvedValue(completed);+ h.persistence.getSkillDeletion.mockResolvedValue(completed);+ h.persistence.claimSkillDeletion.mockResolvedValue(false);+ await expect(h.service.deleteSkillById(h.skillId)).resolves.toMatchObject({+ found: true,+ skillAbsent: true,+ cleanupComplete: true,+ });+ expect(h.deleteFile).not.toHaveBeenCalled();+});++it('preserves a completed deletion result when finalization is later disabled', async () => {+ const h = claimedServiceHarness();+ const completed = { skillId: h.skillId, completedAt: new Date() };+ h.persistence.beginSkillDeletion.mockResolvedValue(completed);+ h.loadConfig.mockResolvedValue({+ skillDeletion: { finalizationEnabled: false },+ } as unknown as AppConfig);++ await expect(h.service.deleteSkillById(h.skillId)).resolves.toMatchObject({+ found: true,+ deleted: true,+ skillAbsent: true,+ cleanupComplete: true,+ });+ await expect(h.service.deleteSkillForLegacy(h.skillId)).resolves.toMatchObject({+ deleted: true,+ skillAbsent: true,+ cleanupComplete: true,+ });+ expect(h.persistence.claimSkillDeletion).not.toHaveBeenCalled();+ expect(h.persistence.finalizeSkillDeletion).not.toHaveBeenCalled();+});++it('does not delete a file row after the heartbeat reports a lost claim', async () => {+ jest.useFakeTimers();+ try {+ const h = claimedServiceHarness();+ let releaseBlob: (() => void) | undefined;+ let startedBlob: (() => void) | undefined;+ const started = new Promise<void>((resolve) => {+ startedBlob = resolve;+ });+ h.deleteFile.mockImplementation(+ () =>+ new Promise<void>((resolve) => {+ releaseBlob = resolve;+ startedBlob?.();+ }),+ );+ h.persistence.refreshSkillDeletionClaim+ .mockResolvedValueOnce(true)+ .mockResolvedValueOnce(true)+ .mockResolvedValueOnce(false);+ const deletion = h.service.deleteSkillById(h.skillId);+ await started;+ await jest.advanceTimersByTimeAsync(30_000);+ releaseBlob?.();+ await expect(deletion).resolves.toMatchObject({ found: true, cleanupComplete: false });+ expect(h.persistence.deleteSkillFileAfterBlob).not.toHaveBeenCalled();+ expect(h.persistence.finalizeSkillDeletion).not.toHaveBeenCalled();+ } finally {+ jest.useRealTimers();+ }+});diff --git a/packages/api/src/skills/deletionService.ts b/packages/api/src/skills/deletionService.tsnew file mode 100644index 0000000000..d6a00359b1--- /dev/null+++ b/packages/api/src/skills/deletionService.ts@@ -0,0 +1,474 @@+import { randomUUID } from 'node:crypto';+import { DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY } from 'librechat-data-provider';+import type {+ AppConfig,+ DeleteSkillResult,+ SkillDeletionRecord,+ SkillUploadIntentRecord,+} from '@librechat/data-schemas';+import type { SkillDeletionDeps, SkillDeletionProgress } from './deletion';+import type { ServerRequest, StrategyFunctions } from '~/types';+import { runSkillDeletion, SkillDeletionClaimLostError } from './deletion';+import { createConcurrencyLimiter } from '~/utils/promise';+import { getStorageMetadata } from '~/storage/metadata';+import { getSafeErrorMetadata } from '~/utils/errors';++type Persistence = Omit<+ SkillDeletionDeps,+ 'deleteBlob' | 'deleteReservedBlob' | 'completeSkillDeletion'+> & {+ completeSkillDeletion: (id: string, owner: string) => Promise<boolean>;+ discoverUnjournaledDeletions: (limit: number) => Promise<number>;+ listSkillDeletionTombstones: (limit: number) => Promise<SkillDeletionRecord[]>;+ claimSkillDeletion: (id: string, owner: string, leaseMs: number) => Promise<boolean>;+ refreshSkillDeletionClaim: (id: string, owner: string, leaseMs: number) => Promise<boolean>;+ noteSkillDeletionAttempt: (id: string) => Promise<void>;+ releaseSkillDeletionClaim: (id: string, owner: string) => Promise<void>;+ scheduleSkillDeletionRecheck: (id: string, owner: string, delayMs: number) => Promise<boolean>;+ listAbandonedSkillUploadIntents: (limit: number) => Promise<SkillUploadIntentRecord[]>;+ claimAbandonedSkillUploadIntent: (+ fileId: string,+ owner: string,+ leaseMs: number,+ ) => Promise<boolean>;+ refreshAbandonedSkillUploadIntentClaim: (+ fileId: string,+ owner: string,+ leaseMs: number,+ ) => Promise<boolean>;+ noteAbandonedSkillUploadIntentAttempt: (fileId: string) => Promise<void>;+ releaseAbandonedSkillUploadIntentClaim: (fileId: string, owner: string) => Promise<void>;+ scheduleAbandonedSkillUploadIntentRecheck: (+ fileId: string,+ owner: string,+ delayMs: number,+ ) => Promise<boolean>;+ skillFileExistsForIntent: (skillId: string, fileId: string) => Promise<boolean>;+};++interface SkillDeletionServiceDeps {+ persistence: Persistence;+ loadConfig: () => Promise<AppConfig | undefined>;+ getStrategyFunctions: (source: string) => Partial<StrategyFunctions>;+ runAsSystem: <T>(callback: () => Promise<T>) => Promise<T>;+ logger: Pick<Console, 'error' | 'warn'>;+}++export interface SkillDeletionSweepOptions {+ sweepIntervalMs: number;+ claimLeaseMs: number;+ batchSize: number;+}++export interface SkillDeletionService {+ deleteSkillById: (skillId: string) => Promise<SkillDeletionProgress>;+ deleteSkillForLegacy: (+ skillId: string,+ ) => Promise<DeleteSkillResult & { blobCleanupManaged: boolean }>;+ startSkillDeletionSweep: (options?: Partial<SkillDeletionSweepOptions>) => {+ stop: () => void;+ runOnce: () => Promise<void>;+ };+}++type CleanupLimiter = ReturnType<typeof createConcurrencyLimiter>;++/** The legacy server supplies storage, database, and context dependencies; it owns no worker logic. */+export function createSkillDeletionService(deps: SkillDeletionServiceDeps): SkillDeletionService {+ async function deleteStoredBlob(+ file: {+ filepath: string;+ source: string;+ storageKey?: string | null;+ storageRegion?: string | null;+ author: string;+ tenantId?: string;+ },+ config: AppConfig | undefined,+ ) {+ const strategy = deps.getStrategyFunctions(file.source);+ if (!strategy.deleteFile) {+ throw new Error(`Skill storage backend ${file.source} has no deletion strategy`);+ }+ const ownerId = file.author;+ await strategy.deleteFile(+ {+ config,+ user: { id: ownerId, _id: ownerId, tenantId: file.tenantId },+ } as unknown as ServerRequest,+ {+ filepath: file.filepath,+ storageKey: file.storageKey,+ storageRegion: file.storageRegion,+ user: ownerId,+ tenantId: file.tenantId,+ },+ );+ }++ function pendingDeletion(found: boolean): SkillDeletionProgress {+ return {+ found,+ deleted: false,+ skillAbsent: !found,+ cleanupComplete: !found,+ failedSteps: [],+ };+ }++ function completedDeletion(): SkillDeletionProgress {+ return {+ found: true,+ deleted: true,+ skillAbsent: true,+ cleanupComplete: true,+ failedSteps: [],+ };+ }++ /** Each request or sweep item owns a distinct claim; same-owner reentry would bypass exclusion. */+ async function deleteWithClaim(+ skillId: string,+ claimLeaseMs: number,+ loadedConfig: AppConfig | undefined | null = null,+ limitBlob?: CleanupLimiter,+ ): Promise<SkillDeletionProgress> {+ const journal =+ (await deps.persistence.beginSkillDeletion(skillId)) ??+ (await deps.persistence.getSkillDeletion(skillId));+ if (!journal) return pendingDeletion(false);+ const config = loadedConfig === null ? await deps.loadConfig() : loadedConfig;+ if (config?.skillDeletion?.finalizationEnabled !== true) {+ return journal.completedAt ? completedDeletion() : pendingDeletion(true);+ }+ const owner = randomUUID();+ if (!(await deps.persistence.claimSkillDeletion(skillId, owner, claimLeaseMs))) {+ // A completed tombstone may be deliberately deferred until its next cold audit.+ // Re-read it: another worker could have reopened it after the first lookup.+ if ((await deps.persistence.getSkillDeletion(skillId))?.completedAt) {+ return completedDeletion();+ }+ return pendingDeletion(true);+ }+ let lost = false;+ const assertClaim = async () => {+ if (lost) throw new SkillDeletionClaimLostError();+ try {+ const retained = await deps.persistence.refreshSkillDeletionClaim(+ skillId,+ owner,+ claimLeaseMs,+ );+ if (!retained) {+ lost = true;+ throw new SkillDeletionClaimLostError();+ }+ } catch (error) {+ lost = true;+ throw error;+ }+ };+ const heartbeat = setInterval(() => {+ void assertClaim().catch((error) =>+ deps.logger.error('[skill deletion] Claim heartbeat failed:', getSafeErrorMetadata(error)),+ );+ }, claimLeaseMs / 4);+ heartbeat.unref?.();+ try {+ const progress = await runSkillDeletion(+ skillId,+ {+ ...deps.persistence,+ completeSkillDeletion: (id) => deps.persistence.completeSkillDeletion(id, owner),+ deleteBlob: (file) =>+ limitBlob+ ? limitBlob(() => deleteStoredBlob(file, config ?? undefined))+ : deleteStoredBlob(file, config ?? undefined),+ deleteReservedBlob: (intent) => {+ const deleteIntentBlob = () =>+ deleteStoredBlob(+ {+ filepath: intent.storageKey,+ source: intent.source,+ ...getStorageMetadata({ filepath: intent.storageKey, source: intent.source }),+ author: intent.author,+ tenantId: intent.tenantId,+ },+ config ?? undefined,+ );+ return limitBlob ? limitBlob(deleteIntentBlob) : deleteIntentBlob();+ },+ },+ config?.fileConfig?.skills?.deletionCleanupConcurrency,+ assertClaim,+ journal,+ );+ if (progress.cleanupComplete) {+ await assertClaim();+ const scheduled = await deps.persistence.scheduleSkillDeletionRecheck(+ skillId,+ owner,+ config?.skillDeletion?.recheckIntervalMs ?? 3_600_000,+ );+ if (!scheduled) {+ lost = true;+ throw new SkillDeletionClaimLostError();+ }+ }+ return progress;+ } catch (error) {+ if (error instanceof SkillDeletionClaimLostError) return pendingDeletion(true);+ throw error;+ } finally {+ clearInterval(heartbeat);+ if (!lost) {+ try {+ await assertClaim();+ await deps.persistence.noteSkillDeletionAttempt(skillId);+ } catch (error) {+ deps.logger.error(+ `[skill deletion] Attempt timestamp failed for ${skillId}:`,+ getSafeErrorMetadata(error),+ );+ }+ }+ try {+ await deps.persistence.releaseSkillDeletionClaim(skillId, owner);+ } catch (error) {+ deps.logger.error(+ `[skill deletion] Claim release failed for ${skillId}:`,+ getSafeErrorMetadata(error),+ );+ }+ }+ }++ async function deleteSkillById(skillId: string) {+ const config = await deps.loadConfig();+ return deleteWithClaim(skillId, config?.skillDeletion?.claimLeaseMs ?? 120_000, config);+ }++ async function deleteSkillForLegacy(skillId: string) {+ const progress = await deleteSkillById(skillId);+ return {+ deleted: progress.deleted,+ skillAbsent: progress.skillAbsent,+ cleanupComplete: progress.cleanupComplete,+ failedCleanupSteps: progress.failedSteps,+ blobCleanupManaged: true,+ };+ }++ /** Completed tombstones remain eligible for cold audits: delayed writers may surface later. */+ function startSkillDeletionSweep(options: Partial<SkillDeletionSweepOptions> = {}) {+ const { sweepIntervalMs = 60_000, claimLeaseMs = 120_000, batchSize = 100 } = options;+ let running = false;+ async function sweepAbandonedIntents() {+ const intents = await deps.persistence.listAbandonedSkillUploadIntents(batchSize);+ if (intents.length === 0) return;+ const config = await deps.loadConfig();+ const recheckIntervalMs = config?.skillDeletion?.recheckIntervalMs ?? 3_600_000;+ const limit = createConcurrencyLimiter(+ config?.fileConfig?.skills?.deletionCleanupConcurrency ??+ DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY,+ );+ const results = await Promise.allSettled(+ intents.map((intent) =>+ limit(async () => {+ const { fileId } = intent;+ const owner = randomUUID();+ if (+ !(await deps.persistence.claimAbandonedSkillUploadIntent(fileId, owner, claimLeaseMs))+ ) {+ return;+ }+ let lost = false;+ const assertClaim = async () => {+ if (lost) throw new SkillDeletionClaimLostError();+ try {+ const retained = await deps.persistence.refreshAbandonedSkillUploadIntentClaim(+ fileId,+ owner,+ claimLeaseMs,+ );+ if (!retained) {+ lost = true;+ throw new SkillDeletionClaimLostError();+ }+ } catch (error) {+ lost = true;+ throw error;+ }+ };+ const heartbeat = setInterval(() => {+ void assertClaim().catch((error) =>+ deps.logger.error(+ '[skill upload intent] Claim heartbeat failed:',+ getSafeErrorMetadata(error),+ ),+ );+ }, claimLeaseMs / 4);+ heartbeat.unref?.();+ try {+ const skillId = intent.skillId;+ await assertClaim();+ if (await deps.persistence.getSkillDeletion(skillId)) {+ // Whole-Skill deletion owns intents for this Skill.+ await assertClaim();+ if (+ !(await deps.persistence.scheduleAbandonedSkillUploadIntentRecheck(+ fileId,+ owner,+ recheckIntervalMs,+ ))+ ) {+ lost = true;+ throw new SkillDeletionClaimLostError();+ }+ return;+ }+ await assertClaim();+ if (await deps.persistence.skillFileExistsForIntent(skillId, fileId)) {+ deps.logger.warn(+ `[skill upload intent] ${fileId} still has a file row; retaining blob`,+ );+ await assertClaim();+ if (+ !(await deps.persistence.scheduleAbandonedSkillUploadIntentRecheck(+ fileId,+ owner,+ recheckIntervalMs,+ ))+ ) {+ lost = true;+ throw new SkillDeletionClaimLostError();+ }+ return;+ }+ await assertClaim();+ await deleteStoredBlob(+ {+ filepath: intent.storageKey,+ source: intent.source,+ ...getStorageMetadata({ filepath: intent.storageKey, source: intent.source }),+ author: intent.author,+ tenantId: intent.tenantId,+ },+ config,+ );+ await assertClaim();+ await deps.persistence.markSkillUploadIntentCleaned(fileId);+ await assertClaim();+ if (+ !(await deps.persistence.scheduleAbandonedSkillUploadIntentRecheck(+ fileId,+ owner,+ recheckIntervalMs,+ ))+ ) {+ lost = true;+ throw new SkillDeletionClaimLostError();+ }+ } catch (error) {+ deps.logger.error(+ `[skill upload intent] Cleanup failed for ${fileId}:`,+ getSafeErrorMetadata(error),+ );+ } finally {+ clearInterval(heartbeat);+ if (!lost) {+ try {+ await assertClaim();+ await deps.persistence.noteAbandonedSkillUploadIntentAttempt(fileId);+ } catch (error) {+ deps.logger.error(+ `[skill upload intent] Attempt timestamp failed for ${fileId}:`,+ getSafeErrorMetadata(error),+ );+ }+ }+ try {+ await deps.persistence.releaseAbandonedSkillUploadIntentClaim(fileId, owner);+ } catch (error) {+ deps.logger.error(+ `[skill upload intent] Claim release failed for ${fileId}:`,+ getSafeErrorMetadata(error),+ );+ }+ }+ }),+ ),+ );+ for (const result of results) {+ if (result.status === 'rejected') {+ deps.logger.error(+ '[skill upload intent] Sweep item failed:',+ getSafeErrorMetadata(result.reason),+ );+ }+ }+ }+ async function tick() {+ if (running) return;+ running = true;+ try {+ await deps.runAsSystem(async () => {+ try {+ await deps.persistence.discoverUnjournaledDeletions(batchSize);+ const tombstones = await deps.persistence.listSkillDeletionTombstones(batchSize);+ const config = await deps.loadConfig();+ if (config?.skillDeletion?.finalizationEnabled === true) {+ const concurrency =+ config?.fileConfig?.skills?.deletionCleanupConcurrency ??+ DEFAULT_SKILL_DELETION_CLEANUP_CONCURRENCY;+ // Bound both active tombstones and their aggregate provider calls;+ // each Skill also bounds its own file fanout in runSkillDeletion.+ const limitTombstones = createConcurrencyLimiter(concurrency);+ const limitBlob = createConcurrencyLimiter(concurrency);+ await Promise.all(+ tombstones.map((tombstone) =>+ limitTombstones(async () => {+ const skillId = tombstone.skillId;+ try {+ const progress = await deleteWithClaim(+ skillId,+ claimLeaseMs,+ config,+ limitBlob,+ );+ if (!progress.cleanupComplete) {+ // A live upload is never timed out automatically. Operators must verify+ // its writer is dead before explicitly abandoning the intent.+ deps.logger.warn(+ `[skill deletion] ${skillId} pending manual/retry cleanup: ${progress.failedSteps.join(', ')}`,+ );+ }+ } catch (error) {+ deps.logger.error(+ `[skill deletion] Cleanup failed for ${skillId}:`,+ getSafeErrorMetadata(error),+ );+ }+ }),+ ),+ );+ }+ } finally {+ // Active Skills can own abandoned upload intents with no deletion tombstone.+ await sweepAbandonedIntents();+ }+ });+ } catch (error) {+ deps.logger.error('[skill deletion] Sweep failed:', getSafeErrorMetadata(error));+ } finally {+ running = false;+ }+ }+ const interval = setInterval(tick, sweepIntervalMs);+ interval.unref?.();+ void tick();+ return { stop: () => clearInterval(interval), runOnce: tick };+ }++ return { deleteSkillById, deleteSkillForLegacy, startSkillDeletionSweep };+}diff --git a/packages/data-schemas/src/methods/skill.ts b/packages/data-schemas/src/methods/skill.tsindex 8c25dda640..2d47244d5f 100644--- a/packages/data-schemas/src/methods/skill.ts+++ b/packages/data-schemas/src/methods/skill.ts@@ -44,7 +44,11 @@ export type ValidationIssue = {severity?: 'error' | 'warning';};-export type DeleteSkillCleanupStep = 'agent_allowlists' | 'skill_files' | 'permissions';+export type DeleteSkillCleanupStep =+ | 'agent_allowlists'+ | 'skill_files'+ | 'permissions'+ | 'upload_intents';export type DeleteSkillResult = {/** Whether this call removed the Skill row. */