Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

20.6 KB
Raw Blame
import type { IScheduleRun, ScheduleMethods } from '@librechat/data-schemas';
import type { JobState } from './types';
import { startScheduleErasureSweep } from './erasure';

const oldRun = (overrides: Partial<IScheduleRun> = {}): IScheduleRun =>
  ({
    scheduleId: 'schedule-1',
    user: 'user-1',
    scheduledFor: new Date('2026-08-17T12:00:00.000Z'),
    firedAt: new Date(Date.now() - 31 * 60_000),
    conversationId: 'conversation-1',
    status: 'started',
    ...overrides,
  }) as unknown as IScheduleRun;

async function sweepOnce(options: {
  canInferOwnerDeathFromMissingJob: boolean;
  job?: JobState | null;
}) {
  const methods = {
    getDeletingSchedules: jest.fn(async () => [{ id: 'schedule-1' }]),
    getActiveRunsForSchedule: jest.fn(async () => [oldRun()]),
    recordRunOutcome: jest.fn(async () => undefined),
    eraseScheduleIfDrained: jest.fn(async () => false),
    markEraseAttempted: jest.fn(async () => undefined),
    getRunsForReconciliation: jest.fn(async () => []),
    getUnbookkeptRuns: jest.fn(async () => []),
    finalizeBookkeeping: jest.fn(async () => undefined),
    markRunsReconciled: jest.fn(async () => undefined),
  };
  const sweep = startScheduleErasureSweep({
    methods: methods as unknown as ScheduleMethods,
    getJobStatus: jest.fn(async () => options.job ?? null),
    getTriggerDelivery: jest.fn(async () => null),
    abortScheduledJob: jest.fn(async () => true),
    clearReconciledJob: jest.fn(async () => undefined),
    canInferOwnerDeathFromMissingJob: options.canInferOwnerDeathFromMissingJob,
  });
  await jest.advanceTimersByTimeAsync(5 * 60_000);
  sweep.stop();
  return methods;
}

describe('schedule erasure fallback owner-death evidence', () => {
  beforeEach(() => {
    jest.useFakeTimers();
    jest.spyOn(Math, 'random').mockReturnValue(0);
  });

  afterEach(() => {
    jest.useRealTimers();
    jest.restoreAllMocks();
  });

  it('drives held-job acknowledgements even when no Mongo run is active or unbookkept', async () => {
    const reconcileRetainedJobs = jest.fn(async () => undefined);
    const methods = {
      getDeletingSchedules: jest.fn(async () => []),
      getRunsForReconciliation: jest.fn(async () => []),
      getUnbookkeptRuns: jest.fn(async () => []),
      markRunsReconciled: jest.fn(async () => undefined),
      markEraseAttempted: jest.fn(async () => undefined),
    };
    const sweep = startScheduleErasureSweep({
      methods: methods as unknown as ScheduleMethods,
      getJobStatus: jest.fn(async () => null),
      getTriggerDelivery: jest.fn(async () => null),
      abortScheduledJob: jest.fn(async () => true),
      clearReconciledJob: jest.fn(async () => undefined),
      canInferOwnerDeathFromMissingJob: false,
      reconcileRetainedJobs,
    });
    await jest.advanceTimersByTimeAsync(5 * 60_000);
    sweep.stop();
    expect(reconcileRetainedJobs).toHaveBeenCalledTimes(1);
  });

  it('does not settle a peer-owned run from process-local job absence', async () => {
    const methods = await sweepOnce({ canInferOwnerDeathFromMissingJob: false });

    expect(methods.recordRunOutcome).not.toHaveBeenCalled();
    expect(methods.eraseScheduleIfDrained).toHaveBeenCalledWith('schedule-1');
  });

  it('retains owner-death recovery when job absence is deployment-authoritative', async () => {
    const methods = await sweepOnce({ canInferOwnerDeathFromMissingJob: true });

    expect(methods.recordRunOutcome).toHaveBeenCalledWith(
      expect.objectContaining({
        scheduleId: 'schedule-1',
        status: 'interrupted',
        error: 'Schedule deleted',
      }),
    );
  });

  it('still trusts an identity-matched terminal job in an unsafe topology', async () => {
    const methods = await sweepOnce({
      canInferOwnerDeathFromMissingJob: false,
      job: {
        status: 'complete',
        scheduleId: 'schedule-1',
        scheduledFor: '2026-08-17T12:00:00.000Z',
      },
    });

    expect(methods.recordRunOutcome).toHaveBeenCalledWith(
      expect.objectContaining({ scheduleId: 'schedule-1', status: 'success' }),
    );
  });
});

const DEFINITE = {
  code: 'rejected',
  message: 'rejected before running',
  certainty: 'definite' as const,
};

describe('topology-safe dead-delivery convergence', () => {
  beforeEach(() => {
    jest.useFakeTimers();
    jest.spyOn(Math, 'random').mockReturnValue(0);
  });

  afterEach(() => {
    jest.useRealTimers();
    jest.restoreAllMocks();
  });

  async function convergeOnce(options: {
    run?: Partial<IScheduleRun>;
    delivery?: {
      status: string;
      lastError?: { code: string; message: string; certainty: 'definite' | 'ambiguous' };
    } | null;
    job?: JobState | null;
    canInferOwnerDeathFromMissingJob?: boolean;
  }) {
    const methods = {
      getDeletingSchedules: jest.fn(async () => []),
      getActiveRunsForSchedule: jest.fn(async () => []),
      recordRunOutcome: jest.fn(async () => undefined),
      eraseScheduleIfDrained: jest.fn(async () => false),
      markEraseAttempted: jest.fn(async () => undefined),
      getRunsForReconciliation: jest.fn(async () => [
        oldRun({ deliveryKey: 'dk-1', ...options.run } as Partial<IScheduleRun>),
      ]),
      getUnbookkeptRuns: jest.fn(async () => []),
      finalizeBookkeeping: jest.fn(async () => undefined),
      markRunsReconciled: jest.fn(async () => undefined),
    };
    const getTriggerDelivery = jest.fn(async () => options.delivery ?? null);
    const clearReconciledJob = jest.fn(async () => undefined);
    const sweep = startScheduleErasureSweep({
      methods: methods as unknown as ScheduleMethods,
      getJobStatus: jest.fn(async () => options.job ?? null),
      getTriggerDelivery: getTriggerDelivery as never,
      clearReconciledJob,
      abortScheduledJob: jest.fn(async () => true),
      // Defaults to the UNSAFE topology to prove this path never depends on it.
      canInferOwnerDeathFromMissingJob: options.canInferOwnerDeathFromMissingJob ?? false,
    });
    await jest.advanceTimersByTimeAsync(5 * 60_000);
    sweep.stop();
    return { methods, getTriggerDelivery, clearReconciledJob };
  }

  it('settles a dead delivery as error even in an unsafe topology (positive evidence)', async () => {
    const { methods } = await convergeOnce({
      delivery: {
        status: 'dead',
        lastError: { code: 'blocked', message: 'blocked by moderation', certainty: 'definite' },
      },
    });
    expect(methods.recordRunOutcome).toHaveBeenCalledWith(
      expect.objectContaining({
        scheduleId: 'schedule-1',
        status: 'error',
        error: 'blocked by moderation',
      }),
    );
  });

  it('leaves a live delivery alone', async () => {
    const { methods } = await convergeOnce({ delivery: { status: 'pending' } });
    expect(methods.recordRunOutcome).not.toHaveBeenCalled();
  });

  it('never settles while an identity-matched generation still owns the reservation', async () => {
    const { methods, getTriggerDelivery } = await convergeOnce({
      delivery: { status: 'dead', lastError: DEFINITE },
      job: {
        status: 'running',
        scheduleId: 'schedule-1',
        scheduledFor: '2026-08-17T12:00:00.000Z',
      } as unknown as JobState,
    });
    expect(getTriggerDelivery).not.toHaveBeenCalled();
    expect(methods.recordRunOutcome).not.toHaveBeenCalled();
  });

  it('defers a run whose abort is still in flight', async () => {
    const { methods } = await convergeOnce({
      run: { abortRequestedAt: new Date() } as Partial<IScheduleRun>,
      delivery: { status: 'dead', lastError: DEFINITE },
    });
    expect(methods.recordRunOutcome).not.toHaveBeenCalled();
  });

  /**
   * The gap this closes: a generation that FINISHED but whose owner exhausted every Mongo
   * outcome retry leaves a preserved terminal job as the only evidence. The armed engine's
   * reconciler replays that; the clustered entrypoint arms no engine, so before this the run
   * kept its `started` row — and its global capacity slot — until the job store expired it.
   */
  it('settles a live schedule from its retained terminal job, then releases the evidence', async () => {
    const { methods, clearReconciledJob } = await convergeOnce({
      job: {
        status: 'complete',
        scheduleId: 'schedule-1',
        scheduledFor: '2026-08-17T12:00:00.000Z',
      } as unknown as JobState,
    });
    expect(methods.recordRunOutcome).toHaveBeenCalledWith(
      expect.objectContaining({
        scheduleId: 'schedule-1',
        status: 'success',
        conversationId: 'conversation-1',
      }),
    );
    expect(clearReconciledJob).toHaveBeenCalledWith('conversation-1', {
      scheduleId: 'schedule-1',
      scheduledFor: new Date('2026-08-17T12:00:00.000Z'),
    });
  });

  /** Presence of an identity-matched job is positive evidence in EVERY topology, so this
   *  must not depend on the owner-death inference the unsafe fallback refuses. */
  it.each(['requires_action', 'aborted', 'complete', 'error'] as const)(
    'preserves job-only bearer diagnosis in clustered %s recovery',
    async (status) => {
      const denial = {
        server: 'Files',
        status: 'mcp_permission_denied',
        reason: 'tool_policy_denied',
        recovery: 'restore_permission',
        automaticReplay: false,
        detail: 'unattended_auth_required',
      };
      const { methods, clearReconciledJob } = await convergeOnce({
        job: {
          status,
          scheduleId: 'schedule-1',
          scheduledFor: '2026-08-17T12:00:00.000Z',
          scheduleOutcomeError: `mcp_permission_denied: ${JSON.stringify([denial])}`,
        },
      });
      expect(methods.recordRunOutcome).toHaveBeenCalledWith(
        expect.objectContaining({ status: 'error', mcp: [denial] }),
      );
      expect(clearReconciledJob).toHaveBeenCalled();
      expect(methods.recordRunOutcome.mock.invocationCallOrder[0]).toBeLessThan(
        clearReconciledJob.mock.invocationCallOrder[0],
      );
    },
  );

  it.each(['terminalPersistencePending', 'providerDrained', 'terminalHostActionPending'] as const)(
    'defers clustered denial settlement behind %s',
    async (fence) => {
      const { methods, clearReconciledJob } = await convergeOnce({
        job: {
          status: 'requires_action',
          scheduleId: 'schedule-1',
          scheduledFor: '2026-08-17T12:00:00.000Z',
          [fence]: fence !== 'providerDrained',
          scheduleOutcomeError:
            'mcp_permission_denied: [{"server":"Files","status":"mcp_permission_denied","reason":"tool_policy_denied","detail":"unattended_auth_required"}]',
        },
      });
      expect(methods.recordRunOutcome).not.toHaveBeenCalled();
      expect(clearReconciledJob).not.toHaveBeenCalled();
    },
  );

  it('honors the owner-stamped outcome over the generic terminal status', async () => {
    const { methods } = await convergeOnce({
      job: {
        status: 'complete',
        scheduleId: 'schedule-1',
        scheduledFor: '2026-08-17T12:00:00.000Z',
        scheduleOutcome: 'skipped_balance',
      } as unknown as JobState,
    });
    expect(methods.recordRunOutcome).toHaveBeenCalledWith(
      expect.objectContaining({ status: 'skipped_balance' }),
    );
  });

  it('carries the stamped failure reason from a retained error job', async () => {
    const { methods } = await convergeOnce({
      job: {
        status: 'error',
        scheduleId: 'schedule-1',
        scheduledFor: '2026-08-17T12:00:00.000Z',
        scheduleOutcomeError: 'provider exploded',
      } as unknown as JobState,
    });
    expect(methods.recordRunOutcome).toHaveBeenCalledWith(
      expect.objectContaining({ status: 'error', error: 'provider exploded' }),
    );
  });

  /** A pre-start abort reserves a conversationId but never creates the conversation, so
   *  projecting it would point the schedule card at a chat that does not exist. */
  it('clears the reserved conversationId when the generation never emitted its created event', async () => {
    const { methods } = await convergeOnce({
      job: {
        status: 'aborted',
        scheduleId: 'schedule-1',
        scheduledFor: '2026-08-17T12:00:00.000Z',
        createdEventEmitted: false,
      } as unknown as JobState,
    });
    expect(methods.recordRunOutcome).toHaveBeenCalledWith(
      expect.objectContaining({ status: 'interrupted', clearConversationId: true }),
    );
    expect(methods.recordRunOutcome).toHaveBeenCalledWith(
      expect.not.objectContaining({ conversationId: 'conversation-1' }),
    );
  });

  /** A terminal job belonging to a REPLACEMENT turn (same conversationId, no scheduled
   *  identity) must never finalize this run, nor have its evidence deleted for it. */
  it("ignores a terminal job that lost this occurrence's identity", async () => {
    const { methods, clearReconciledJob } = await convergeOnce({
      job: { status: 'complete' } as unknown as JobState,
    });
    expect(methods.recordRunOutcome).not.toHaveBeenCalled();
    expect(clearReconciledJob).not.toHaveBeenCalled();
  });

  /**
   * A pause whose `requires_action` projection failed every retry leaves the row `started`
   * while the job is durably paused. The armed reconciler replays that; with no engine the
   * row held its GLOBAL capacity slot forever, since a paused job is not terminal and the
   * dead-delivery path never inspects an identity-matched job.
   */
  it('projects a pause the owner never managed to record, freeing the slot', async () => {
    const { methods, clearReconciledJob } = await convergeOnce({
      job: {
        status: 'requires_action',
        scheduleId: 'schedule-1',
        scheduledFor: '2026-08-17T12:00:00.000Z',
      } as unknown as JobState,
    });
    expect(methods.recordRunOutcome).toHaveBeenCalledWith(
      expect.objectContaining({
        scheduleId: 'schedule-1',
        status: 'requires_action',
        conversationId: 'conversation-1',
        // Every replica runs this sweep, so the write itself must be fenced against a
        // resume that claimed the row after this snapshot was taken — as a staleness
        // CUTOFF, so an abandoned claim still recovers.
        resumeClaimStaleBefore: expect.any(Date),
      }),
    );
    // The job is still LIVE awaiting approval — its evidence must not be released.
    expect(clearReconciledJob).not.toHaveBeenCalled();
  });

  /** A resume claims capacity before flipping the job back to `running`; re-projecting the
   *  pause during that hand-off would release the slot the continuation just took. */
  it('defers a paused job while a resume hand-off is in flight', async () => {
    const { methods } = await convergeOnce({
      run: { resumeClaimedAt: new Date() } as Partial<IScheduleRun>,
      job: {
        status: 'requires_action',
        scheduleId: 'schedule-1',
        scheduledFor: '2026-08-17T12:00:00.000Z',
      } as unknown as JobState,
    });
    expect(methods.recordRunOutcome).not.toHaveBeenCalled();
  });

  it('never settles a terminal job while its abort is still in flight', async () => {
    const { methods, clearReconciledJob } = await convergeOnce({
      run: { abortRequestedAt: new Date() } as Partial<IScheduleRun>,
      job: {
        status: 'aborted',
        scheduleId: 'schedule-1',
        scheduledFor: '2026-08-17T12:00:00.000Z',
      } as unknown as JobState,
    });
    expect(methods.recordRunOutcome).not.toHaveBeenCalled();
    expect(clearReconciledJob).not.toHaveBeenCalled();
  });

  it('ignores a legacy reservation carrying no deliveryKey', async () => {
    const { methods, getTriggerDelivery } = await convergeOnce({
      run: { deliveryKey: undefined } as Partial<IScheduleRun>,
      delivery: { status: 'dead', lastError: DEFINITE },
    });
    expect(getTriggerDelivery).not.toHaveBeenCalled();
    expect(methods.recordRunOutcome).not.toHaveBeenCalled();
  });
});

describe('permanent MCP bookkeeping recovery without an armed scheduler', () => {
  beforeEach(() => {
    jest.useFakeTimers();
    jest.spyOn(Math, 'random').mockReturnValue(0);
  });

  afterEach(() => {
    jest.useRealTimers();
    jest.restoreAllMocks();
  });

  it('replays a half-bookkept MCP run without applying ordinary failure policy', async () => {
    const mcp = [
      {
        server: 'Graph',
        status: 'mcp_configuration_missing' as const,
        detail: 'unattended_auth_required' as const,
      },
    ];
    const permanent = oldRun({ status: 'error', error: 'MCP unavailable', mcp });
    const ordinary = oldRun({
      scheduleId: 'schedule-2',
      status: 'error',
      error: 'Temporary upstream failure',
    });
    const methods = {
      getDeletingSchedules: jest.fn(async () => []),
      getActiveRunsForSchedule: jest.fn(async () => []),
      eraseScheduleIfDrained: jest.fn(async () => false),
      markEraseAttempted: jest.fn(async () => undefined),
      getRunsForReconciliation: jest.fn(async () => []),
      recordRunOutcome: jest.fn(async () => undefined),
      getUnbookkeptRuns: jest.fn(async () => [permanent, ordinary]),
      finalizeBookkeeping: jest.fn(async () => undefined),
      markRunsReconciled: jest.fn(async () => undefined),
    };
    const sweep = startScheduleErasureSweep({
      methods: methods as unknown as ScheduleMethods,
      getJobStatus: jest.fn(async () => null),
      getTriggerDelivery: jest.fn(async () => null),
      abortScheduledJob: jest.fn(async () => true),
      clearReconciledJob: jest.fn(async () => undefined),
      canInferOwnerDeathFromMissingJob: false,
    });
    await jest.advanceTimersByTimeAsync(5 * 60_000);
    sweep.stop();
    expect(methods.finalizeBookkeeping).toHaveBeenCalledTimes(1);
    expect(methods.finalizeBookkeeping).toHaveBeenCalledWith(
      expect.objectContaining({
        scheduleId: 'schedule-1',
        status: 'error',
        mcp,
        autoDisableAfterFailures: Number.MAX_SAFE_INTEGER,
      }),
    );
    expect(methods.markRunsReconciled).toHaveBeenCalledWith([permanent, ordinary]);
  });
});

describe('dead-delivery certainty fence', () => {
  beforeEach(() => {
    jest.useFakeTimers();
    jest.spyOn(Math, 'random').mockReturnValue(0);
  });

  afterEach(() => {
    jest.useRealTimers();
    jest.restoreAllMocks();
  });

  async function sweepWithDelivery(options: {
    certainty: 'definite' | 'ambiguous';
    canInferOwnerDeathFromMissingJob: boolean;
  }) {
    const methods = {
      getDeletingSchedules: jest.fn(async () => []),
      getActiveRunsForSchedule: jest.fn(async () => []),
      recordRunOutcome: jest.fn(async () => undefined),
      eraseScheduleIfDrained: jest.fn(async () => false),
      markEraseAttempted: jest.fn(async () => undefined),
      getRunsForReconciliation: jest.fn(async () => [oldRun({ deliveryKey: 'dk-1' } as never)]),
      getUnbookkeptRuns: jest.fn(async () => []),
      finalizeBookkeeping: jest.fn(async () => undefined),
      markRunsReconciled: jest.fn(async () => undefined),
    };
    const sweep = startScheduleErasureSweep({
      methods: methods as unknown as ScheduleMethods,
      getJobStatus: jest.fn(async () => null),
      getTriggerDelivery: (async () => ({
        status: 'dead',
        lastError: { code: 'x', message: 'timed out', certainty: options.certainty },
      })) as never,
      abortScheduledJob: jest.fn(async () => true),
      clearReconciledJob: jest.fn(async () => undefined),
      canInferOwnerDeathFromMissingJob: options.canInferOwnerDeathFromMissingJob,
    });
    await jest.advanceTimersByTimeAsync(5 * 60_000);
    sweep.stop();
    return methods;
  }

  /**
   * `dead` is not proof of rejection: the trigger host marks response timeouts and invalid
   * success responses `ambiguous`, and the engine dead-letters those after exhausting
   * retries — so an ambiguous dead letter can sit over a generation a peer accepted and is
   * still running. Settling it here would release capacity mid-run.
   */
  it('refuses an AMBIGUOUS dead letter when absence is not deployment-authoritative', async () => {
    const methods = await sweepWithDelivery({
      certainty: 'ambiguous',
      canInferOwnerDeathFromMissingJob: false,
    });
    expect(methods.recordRunOutcome).not.toHaveBeenCalled();
  });

  it('settles an ambiguous dead letter once job absence IS deployment-authoritative', async () => {
    const methods = await sweepWithDelivery({
      certainty: 'ambiguous',
      canInferOwnerDeathFromMissingJob: true,
    });
    expect(methods.recordRunOutcome).toHaveBeenCalledWith(
      expect.objectContaining({ status: 'error', error: 'timed out' }),
    );
  });

  it('settles a DEFINITE rejection even in an unsafe topology', async () => {
    const methods = await sweepWithDelivery({
      certainty: 'definite',
      canInferOwnerDeathFromMissingJob: false,
    });
    expect(methods.recordRunOutcome).toHaveBeenCalledWith(
      expect.objectContaining({ status: 'error', error: 'timed out' }),
    );
  });
});