Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import mongoose from 'mongoose';
import { logger } from '@librechat/data-schemas';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { HumanMessage } from '@librechat/agents/langchain/messages';
import { MongoDBSaver } from '@langchain/langgraph-checkpoint-mongodb';
import { emptyCheckpoint, ERROR, INTERRUPT } from '@langchain/langgraph-checkpoint';
import {
getAgentCheckpointer,
hasDurableAgentInterruptCheckpoint,
captureAgentCheckpointGeneration,
deleteAgentCheckpoint,
deleteOwnedAgentCheckpoints,
forkAgentEventCheckpoint,
captureAgentEventCheckpoint,
LazyMongoSaver,
CheckpointTooLargeError,
LIBRECHAT_CHECKPOINT_NAMESPACE_KEY,
LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY,
setupCheckpointIndexes,
__resetCheckpointerForTests,
} from './checkpointer';
import { createCheckpointNamespace } from '../stream/checkpoints';
/**
* Integration tests for the durable Mongo checkpointer seam, against a real
* (in-memory) MongoDB. The unit spec covers config/selection with no connection;
* this proves the part that actually matters for correctness — that a checkpoint
* written for a thread can be read back and that `deleteAgentCheckpoint` truly
* prunes it (the cross-turn isolation guarantee), scoped to a single thread.
*/
const MONGO_CFG = { type: 'mongo' as const, ttl: 3600 };
/** Minimal LangGraph put() args for an empty checkpoint under a thread. */
function putArgs(threadId: string, checkpointNamespace = '') {
const config = {
configurable: { thread_id: threadId, checkpoint_ns: checkpointNamespace },
};
const metadata = { source: 'input' as const, step: -1, writes: null, parents: {} };
return { config, checkpoint: emptyCheckpoint(), metadata };
}
const readConfig = (threadId: string, checkpointNamespace = '') => ({
configurable: { thread_id: threadId, checkpoint_ns: checkpointNamespace },
});
/**
* Persist a checkpoint the way LangGraph does on a PAUSE: an interrupt `putWrites` on the
* `INTERRUPT` channel for the checkpoint id, then `put` of that checkpoint. The lazy saver
* persists only checkpoints seeded this way (a bare `put` is a clean exit → discarded).
*/
async function seedInterruptCheckpoint(
saver: MongoDBSaver,
threadId: string,
checkpointNamespace = '',
interruptId = 'interrupt-current',
) {
const { config, checkpoint, metadata } = putArgs(threadId, checkpointNamespace);
await saver.putWrites(
{
configurable: {
thread_id: threadId,
checkpoint_ns: checkpointNamespace,
checkpoint_id: checkpoint.id,
},
},
[[INTERRUPT, { id: interruptId, value: 'approve?' }]],
'task-1',
);
await saver.put(config, checkpoint, metadata);
return checkpoint;
}
let mongoServer: MongoMemoryServer;
beforeAll(async () => {
mongoServer = await MongoMemoryServer.create();
await mongoose.connect(mongoServer.getUri());
}, 60000);
afterAll(async () => {
await mongoose.disconnect();
await mongoServer.stop();
});
beforeEach(() => {
// Force a fresh saver build (+ setup) against the live connection each test.
__resetCheckpointerForTests();
});
afterEach(async () => {
await mongoose.connection.dropDatabase();
});
describe('checkpointer (mongodb-memory-server integration)', () => {
it('builds a real MongoDBSaver when Mongo is connected', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
expect(saver).toBeDefined();
// setup() created the checkpoint collection with a TTL index on `upserted_at`.
const indexes = await mongoose.connection.db!.collection('agent_checkpoints').indexes();
const ttlIndex = indexes.find((idx) => idx.expireAfterSeconds != null);
expect(ttlIndex).toBeDefined();
expect(ttlIndex?.expireAfterSeconds).toBe(3600);
});
it('verifies that an interrupt write has its matching durable checkpoint', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
expect(saver).toBeDefined();
const missingIdentity = {
checkpointId: 'missing-checkpoint',
interruptId: 'interrupt-current',
};
await expect(
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, missingIdentity),
).resolves.toBe(false);
const checkpoint = await seedInterruptCheckpoint(saver!, 'verified-pause');
const identity = {
checkpointId: checkpoint.id,
interruptId: 'interrupt-current',
};
await expect(
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, identity),
).resolves.toBe(true);
await expect(
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, missingIdentity),
).resolves.toBe(false);
await expect(
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, {
...identity,
interruptId: 'interrupt-stale',
}),
).resolves.toBe(false);
await mongoose.connection.db!.collection('agent_checkpoints').deleteMany({
thread_id: 'verified-pause',
});
await expect(
hasDurableAgentInterruptCheckpoint('verified-pause', MONGO_CFG, identity),
).resolves.toBe(false);
});
it("uses Mongoose's selected database instead of the MongoClient URI default", async () => {
await mongoose.disconnect();
await mongoose.connect(mongoServer.getUri('driver_default'), { dbName: 'active_app' });
__resetCheckpointerForTests();
const saver = await getAgentCheckpointer(MONGO_CFG);
expect(saver).toBeDefined();
await seedInterruptCheckpoint(saver!, 'db-selection-pause');
expect(
await mongoose.connection.db!.collection('agent_checkpoints').countDocuments({
thread_id: 'db-selection-pause',
}),
).toBe(1);
expect(
await mongoose.connection
.getClient()
.db('driver_default')
.collection('agent_checkpoints')
.countDocuments({
thread_id: 'db-selection-pause',
}),
).toBe(0);
});
it('returns undefined for the memory type (SDK MemorySaver fallback) even when connected', async () => {
expect(await getAgentCheckpointer({ type: 'memory' })).toBeUndefined();
});
it('memoizes one saver per resolved config', async () => {
const a = await getAgentCheckpointer(MONGO_CFG);
const b = await getAgentCheckpointer(MONGO_CFG);
expect(a).toBe(b);
});
it('persists clean event-actor exits and forks only the committed checkpoint', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `actor-${new mongoose.Types.ObjectId().toString()}`;
const checkpoint = emptyCheckpoint();
const config = {
configurable: {
thread_id: threadId,
checkpoint_ns: '',
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: 'event-actor/base',
[LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY]: 'event-1',
},
};
await saver!.put(config, checkpoint, {
source: 'input',
step: -1,
parents: {},
});
await expect(
captureAgentEventCheckpoint(threadId, 'event-actor/base', 'event-1', MONGO_CFG),
).resolves.toEqual({
threadId,
checkpointId: checkpoint.id,
checkpointNs: 'event-actor/base',
});
await expect(
forkAgentEventCheckpoint(
{ threadId, checkpointId: checkpoint.id, checkpointNs: 'event-actor/base' },
'event-actor/fork',
'event-2',
MONGO_CFG,
),
).resolves.toEqual({
threadId,
checkpointId: checkpoint.id,
checkpointNs: 'event-actor/fork',
});
expect(
await mongoose.connection
.db!.collection('agent_checkpoints')
.countDocuments({ thread_id: threadId }),
).toBe(2);
});
it('replaces checkpoint-carried Skill context while preserving the committed base', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `actor-${new mongoose.Types.ObjectId().toString()}`;
const checkpoint = emptyCheckpoint();
checkpoint.channel_values.messages = [
new HumanMessage({ content: 'ordinary history' }),
new HumanMessage({
content: 'old skill body',
additional_kwargs: { isMeta: true, source: 'skill', skillName: 'analysis' },
}),
];
checkpoint.channel_versions.messages = 1;
const sourceConfig = {
configurable: {
thread_id: threadId,
checkpoint_ns: '',
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: 'event-actor/base',
[LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY]: 'event-1',
},
};
await saver!.put(sourceConfig, checkpoint, {
source: 'input',
step: -1,
parents: {},
});
await forkAgentEventCheckpoint(
{ threadId, checkpointId: checkpoint.id, checkpointNs: 'event-actor/base' },
'event-actor/fork',
'event-2',
MONGO_CFG,
{
source: 'skill',
messages: [
new HumanMessage({
content: 'current skill body',
additional_kwargs: { isMeta: true, source: 'skill', skillName: 'analysis' },
}),
],
},
);
const source = await saver!.getTuple(sourceConfig);
const fork = await saver!.getTuple({
configurable: {
thread_id: threadId,
checkpoint_ns: '',
checkpoint_id: checkpoint.id,
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: 'event-actor/fork',
[LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY]: 'event-2',
},
});
expect(
(source?.checkpoint.channel_values.messages as HumanMessage[]).map(
(message) => message.content,
),
).toEqual(['ordinary history', 'old skill body']);
expect(
(fork?.checkpoint.channel_values.messages as HumanMessage[]).map(
(message) => message.content,
),
).toEqual(['ordinary history', 'current skill body']);
});
it('warm-continues from a copied actor head without mutating the committed base', async () => {
const { StateGraph, START, END, Annotation } = await import('@langchain/langgraph');
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `actor-${new mongoose.Types.ObjectId().toString()}`;
const State = Annotation.Root({
events: Annotation<string[]>({
reducer: (left, right) => [...left, ...right],
default: () => [],
}),
});
const graph = new StateGraph(State)
.addNode('observe', (state: { events: string[] }) => ({
events: [`seen:${state.events[state.events.length - 1]}`],
}))
.addEdge(START, 'observe')
.addEdge('observe', END)
.compile({ checkpointer: saver as never });
const config = (checkpointNamespace: string, invocationId: string, checkpointId?: string) => ({
configurable: {
thread_id: threadId,
checkpoint_ns: '',
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: checkpointNamespace,
[LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY]: invocationId,
...(checkpointId == null ? {} : { checkpoint_id: checkpointId }),
},
durability: 'exit' as const,
});
await graph.invoke({ events: ['event-1'] }, config('event-actor/base', 'event-1'));
const base = await captureAgentEventCheckpoint(
threadId,
'event-actor/base',
'event-1',
MONGO_CFG,
);
expect(base).not.toBeNull();
const fork = await forkAgentEventCheckpoint(base!, 'event-actor/fork', 'event-2', MONGO_CFG);
expect(fork).not.toBeNull();
const warm = await graph.invoke(
{ events: ['event-2'] },
config('event-actor/fork', 'event-2', fork!.checkpointId),
);
expect(warm.events).toEqual(['event-1', 'seen:event-1', 'event-2', 'seen:event-2']);
const committedBase = await saver!.getTuple({
configurable: {
thread_id: threadId,
checkpoint_ns: '',
checkpoint_id: base!.checkpointId,
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: 'event-actor/base',
},
});
expect(committedBase?.checkpoint.channel_values.events).toEqual(['event-1', 'seen:event-1']);
});
it('cold-rebuilds in a fresh namespace when an invalidated head id is not copied', async () => {
const { StateGraph, START, END, Annotation } = await import('@langchain/langgraph');
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `actor-${new mongoose.Types.ObjectId().toString()}`;
const State = Annotation.Root({
events: Annotation<string[]>({
reducer: (left, right) => [...left, ...right],
default: () => [],
}),
});
const graph = new StateGraph(State)
.addNode('observe', (state: { events: string[] }) => ({
events: [`seen:${state.events[state.events.length - 1]}`],
}))
.addEdge(START, 'observe')
.addEdge('observe', END)
.compile({ checkpointer: saver as never });
const config = (checkpointNamespace: string, invocationId: string, checkpointId?: string) => ({
configurable: {
thread_id: threadId,
checkpoint_ns: '',
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: checkpointNamespace,
[LIBRECHAT_EVENT_ACTOR_INVOCATION_KEY]: invocationId,
...(checkpointId == null ? {} : { checkpoint_id: checkpointId }),
},
durability: 'exit' as const,
});
await graph.invoke({ events: ['legacy-before'] }, config('event-actor/base', 'event-1'));
const base = await captureAgentEventCheckpoint(
threadId,
'event-actor/base',
'event-1',
MONGO_CFG,
);
expect(base).not.toBeNull();
const rebuilt = await graph.invoke(
{ events: ['event-after-legacy'] },
config('event-actor/cold', 'event-2', base!.checkpointId),
);
expect(rebuilt.events).toEqual(['event-after-legacy', 'seen:event-after-legacy']);
});
it('deleteAgentCheckpoint prunes a thread’s persisted checkpoint', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
expect(saver).toBeDefined();
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
await seedInterruptCheckpoint(saver!, threadId);
// The checkpoint is durably readable before pruning…
expect(await saver!.getTuple(readConfig(threadId))).toBeDefined();
await deleteAgentCheckpoint(threadId, MONGO_CFG);
// …and gone after (so turn N+1 on the same conversationId can't rehydrate it).
expect(await saver!.getTuple(readConfig(threadId))).toBeUndefined();
});
it('captured legacy cleanup prunes nested graph checkpoints and writes without a thread-wide delete', async () => {
const { StateGraph, START, END, interrupt, Annotation } = await import('@langchain/langgraph');
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const State = Annotation.Root({
origin: Annotation,
approved: Annotation,
});
const child = new StateGraph(State)
.addNode('child-gate', (state: { origin?: string }) => ({
approved: interrupt(`approve-${state.origin}`),
}))
.addEdge(START, 'child-gate')
.addEdge('child-gate', END)
.compile();
const graph = new StateGraph(State)
.addNode('child', child)
.addEdge(START, 'child')
.addEdge('child', END)
.compile({ checkpointer: saver as never });
const config = {
configurable: { thread_id: threadId, checkpoint_ns: '' },
durability: 'exit' as const,
};
await graph.invoke({ origin: 'legacy', approved: null }, config);
const db = mongoose.connection.db!;
const storedNamespaces = await db
.collection('agent_checkpoints')
.distinct('checkpoint_ns', { thread_id: threadId });
expect(storedNamespaces).toContain('');
expect(storedNamespaces.some((namespace) => namespace.startsWith('child:'))).toBe(true);
expect(
await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
).toBeGreaterThan(0);
const generation = await captureAgentCheckpointGeneration(threadId, MONGO_CFG, {
throwOnError: true,
});
const replacement = await seedInterruptCheckpoint(saver!, threadId, '2000');
await expect(
deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, {
throwOnError: true,
checkpointNamespace: '',
}),
).rejects.toThrow('requires a captured checkpoint generation');
await deleteAgentCheckpoint(threadId, MONGO_CFG, generation, { throwOnError: true });
expect(await db.collection('agent_checkpoints').countDocuments({ thread_id: threadId })).toBe(
1,
);
expect(
await db
.collection('agent_checkpoints')
.countDocuments({ thread_id: threadId, checkpoint_ns: { $ne: '2000' } }),
).toBe(0);
expect(
await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
).toBe(1);
expect(await saver!.getTuple(readConfig(threadId, '2000'))).toMatchObject({
checkpoint: { id: replacement.id },
});
});
it('prunes only the targeted thread, leaving other conversations intact', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadA = `convo-${new mongoose.Types.ObjectId().toString()}`;
const threadB = `convo-${new mongoose.Types.ObjectId().toString()}`;
await seedInterruptCheckpoint(saver!, threadA);
await seedInterruptCheckpoint(saver!, threadB);
await deleteAgentCheckpoint(threadA, MONGO_CFG);
expect(await saver!.getTuple(readConfig(threadA))).toBeUndefined();
expect(await saver!.getTuple(readConfig(threadB))).toBeDefined();
});
it('scheduled namespace capture prunes root and nested writes while preserving a replacement', async () => {
const db = mongoose.connection.db!;
const threadId = 'scheduled-owned-run';
const namespace = createCheckpointNamespace('owner', 'tenant');
const replacement = createCheckpointNamespace('owner', 'tenant');
const rows = [namespace, `${namespace}|child`, replacement, ''].map((checkpoint_ns) => ({
thread_id: threadId,
checkpoint_ns,
checkpoint_id: 'same-checkpoint-id',
}));
for (const name of ['agent_checkpoints', 'agent_checkpoint_writes']) {
await db.collection(name).insertMany(rows.map((row) => ({ ...row })));
}
const capture = await captureAgentCheckpointGeneration(threadId, MONGO_CFG, {
checkpointNamespace: namespace,
throwOnError: true,
});
expect(capture?.checkpointIds).toContain('same-checkpoint-id');
await deleteAgentCheckpoint(threadId, MONGO_CFG, capture, { throwOnError: true });
for (const name of ['agent_checkpoints', 'agent_checkpoint_writes']) {
expect(await db.collection(name).distinct('checkpoint_ns', { thread_id: threadId })).toEqual(
expect.arrayContaining([replacement, '']),
);
expect(await db.collection(name).countDocuments({ thread_id: threadId })).toBe(2);
}
});
it('generation-scoped cleanup preserves a replacement checkpoint on the same thread', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const resumed = await seedInterruptCheckpoint(saver!, threadId);
const generation = await captureAgentCheckpointGeneration(threadId, MONGO_CFG);
const replacement = await seedInterruptCheckpoint(saver!, threadId);
await deleteAgentCheckpoint(threadId, MONGO_CFG, generation);
expect(await saver!.getTuple(readConfig(threadId))).toMatchObject({
checkpoint: { id: replacement.id },
});
expect(
await saver!.getTuple({
configurable: {
thread_id: threadId,
checkpoint_ns: '',
checkpoint_id: resumed.id,
},
}),
).toBeUndefined();
expect(
await mongoose.connection
.db!.collection('agent_checkpoint_writes')
.countDocuments({ thread_id: threadId, checkpoint_id: replacement.id }),
).toBe(1);
expect(
await mongoose.connection
.db!.collection('agent_checkpoint_writes')
.countDocuments({ thread_id: threadId, checkpoint_id: resumed.id }),
).toBe(0);
});
it('legacy capture and cleanup preserve ordinary owned payload even with matching checkpoint IDs', async () => {
const saver = (await getAgentCheckpointer(MONGO_CFG))!;
const threadId = 'legacy-owner-collision';
const legacy = await seedInterruptCheckpoint(saver, threadId);
const ownedNamespace = createCheckpointNamespace('other-owner', 'tenant');
const db = mongoose.connection.db!;
for (const name of ['agent_checkpoints', 'agent_checkpoint_writes']) {
const row = await db.collection(name).findOne({ thread_id: threadId });
const { _id, ...payload } = row!;
await db.collection(name).insertOne({ ...payload, checkpoint_ns: ownedNamespace });
await db
.collection(name)
.insertOne({ ...payload, checkpoint_id: 'only-owned', checkpoint_ns: ownedNamespace });
}
const captured = await captureAgentCheckpointGeneration(threadId, MONGO_CFG);
expect(captured.checkpointIds).toEqual([legacy.id]);
await deleteAgentCheckpoint(threadId, MONGO_CFG, captured, { throwOnError: true });
await deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, { throwOnError: true });
for (const name of ['agent_checkpoints', 'agent_checkpoint_writes']) {
const rows = await db.collection(name).find({ thread_id: threadId }).toArray();
expect(rows).toHaveLength(2);
expect(rows.every((row) => row.checkpoint_ns === ownedNamespace)).toBe(true);
}
await deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, {
checkpointNamespace: ownedNamespace,
throwOnError: true,
});
expect(await db.collection('agent_checkpoints').countDocuments({ thread_id: threadId })).toBe(
0,
);
expect(
await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
).toBe(0);
});
it('isolates a fresh generation from a predecessor checkpoint written after replacement', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const predecessorNamespace = '1000';
const replacementNamespace = '2000';
await seedInterruptCheckpoint(saver!, threadId, predecessorNamespace);
// Fresh B prunes only its own namespace before graph construction. A is
// still running remotely and writes another interrupt after that barrier.
await deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, {
checkpointNamespace: replacementNamespace,
});
const latePredecessor = await seedInterruptCheckpoint(saver!, threadId, predecessorNamespace);
expect(await saver!.getTuple(readConfig(threadId, replacementNamespace))).toBeUndefined();
const replacement = await seedInterruptCheckpoint(saver!, threadId, replacementNamespace);
expect(await saver!.getTuple(readConfig(threadId, replacementNamespace))).toMatchObject({
checkpoint: { id: replacement.id },
});
expect(await saver!.getTuple(readConfig(threadId, predecessorNamespace))).toMatchObject({
checkpoint: { id: latePredecessor.id },
});
});
it('adapts LangGraph root namespaces and prevents a late predecessor pause from hydrating into its replacement', async () => {
const { StateGraph, START, END, interrupt, Annotation, Command } = await import(
'@langchain/langgraph'
);
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const State = Annotation.Root({
origin: Annotation,
approved: Annotation,
});
const graph = new StateGraph(State)
.addNode('gate', (state: { origin?: string }) => ({
approved: interrupt(`approve-${state.origin}`),
}))
.addEdge(START, 'gate')
.addEdge('gate', END)
.compile({ checkpointer: saver as never });
const scopedConfig = (generation: string, graphNamespace = '') => ({
configurable: {
thread_id: threadId,
checkpoint_ns: graphNamespace,
[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: generation,
},
durability: 'exit' as const,
});
// Deliberately nonempty on the initial root calls: PregelLoop.initialize
// resets these to ''. The ordinary client wiring uses the normalized
// configs below on both fresh and resume calls.
const replacementStartConfig = scopedConfig('generation-b', 'caller-root-generation-b');
const predecessorStartConfig = scopedConfig('generation-a', 'caller-root-generation-a');
const replacementConfig = scopedConfig('generation-b');
const predecessorConfig = scopedConfig('generation-a');
// B has started and paused. A's remote owner then lands a late pause on
// the same conversation thread; physical namespaces must keep it out of B.
await graph.invoke({ origin: 'B', approved: null }, replacementStartConfig);
await graph.invoke({ origin: 'A', approved: null }, predecessorStartConfig);
const storedNamespaces = await mongoose.connection
.db!.collection('agent_checkpoints')
.distinct('checkpoint_ns', { thread_id: threadId });
expect(new Set(storedNamespaces)).toEqual(new Set(['generation-a', 'generation-b']));
expect(storedNamespaces).not.toContain('');
expect(storedNamespaces).not.toContain('caller-root-generation-a');
expect(storedNamespaces).not.toContain('caller-root-generation-b');
const replacementState = await graph.getState(replacementConfig);
expect(replacementState.values).toMatchObject({ origin: 'B' });
const listedReplacementCheckpoints = [];
for await (const tuple of saver!.list(replacementConfig)) {
listedReplacementCheckpoints.push(tuple);
}
expect(listedReplacementCheckpoints.length).toBeGreaterThan(0);
expect(
listedReplacementCheckpoints.every(
(tuple) =>
tuple.config.configurable?.checkpoint_ns === '' &&
tuple.config.configurable?.[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY] === 'generation-b',
),
).toBe(true);
const replacementResult = await graph.invoke(
new Command({ resume: 'YES-B' }),
replacementConfig,
);
expect(replacementResult).toMatchObject({ origin: 'B', approved: 'YES-B' });
// Terminal B cleanup removes its whole namespace, including any write
// committed after an earlier snapshot, while late A remains resumable.
await deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, {
checkpointNamespace: 'generation-b',
});
expect(await graph.getState(replacementConfig)).toMatchObject({ next: [] });
expect(await graph.getState(predecessorConfig)).toMatchObject({
values: expect.objectContaining({ origin: 'A' }),
next: expect.arrayContaining(['gate']),
});
});
it('deleteAgentCheckpoint is a no-op for an undefined threadId', async () => {
await expect(deleteAgentCheckpoint(undefined, MONGO_CFG)).resolves.toBeUndefined();
});
it('bulk-deletes only an owned generation scope when thread IDs collide', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `collision-${new mongoose.Types.ObjectId().toString()}`;
const ownedNamespace = createCheckpointNamespace('owner', 'tenant');
await seedInterruptCheckpoint(saver!, threadId, ownedNamespace);
await seedInterruptCheckpoint(saver!, threadId, `${ownedNamespace}|subgraph`);
await seedInterruptCheckpoint(saver!, threadId, 'foreign-generation');
await seedInterruptCheckpoint(saver!, threadId, '');
await deleteOwnedAgentCheckpoints('owner', 'tenant', [threadId, threadId], MONGO_CFG);
expect(await saver!.getTuple(readConfig(threadId, ownedNamespace))).toBeUndefined();
expect(
await saver!.getTuple(readConfig(threadId, `${ownedNamespace}|subgraph`)),
).toBeUndefined();
expect(await saver!.getTuple(readConfig(threadId, 'foreign-generation'))).toBeDefined();
expect(await saver!.getTuple(readConfig(threadId, ''))).toBeDefined();
const db = mongoose.connection.db!;
expect(
await db.collection('agent_checkpoint_writes').countDocuments({
thread_id: threadId,
checkpoint_ns: { $in: [ownedNamespace, `${ownedNamespace}|subgraph`] },
}),
).toBe(0);
expect(
await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
).toBe(2);
});
it('propagates a bulk checkpoint deletion failure for retry', async () => {
const deletion = jest
.spyOn(mongoose.mongo.Collection.prototype, 'deleteMany')
.mockRejectedValueOnce(new Error('checkpoint deletion unavailable'));
try {
await expect(
deleteOwnedAgentCheckpoints('owner', 'tenant', ['conversation-1'], MONGO_CFG),
).rejects.toThrow('checkpoint deletion unavailable');
} finally {
deletion.mockRestore();
}
});
});
describe('LazyMongoSaver (lazy persistence — mongodb-memory-server)', () => {
it('does NOT persist a clean-exit checkpoint (a bare put with no pending writes)', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const { config, checkpoint, metadata } = putArgs(threadId);
// A non-paused run's exit put — no preceding putWrites.
await saver!.put(config, checkpoint, metadata);
expect(await saver!.getTuple(readConfig(threadId))).toBeUndefined();
const count = await mongoose.connection
.db!.collection('agent_checkpoints')
.countDocuments({ thread_id: threadId });
expect(count).toBe(0);
});
it('persists a checkpoint anchored by a NON-interrupt write (delta-channel safety)', async () => {
// K1/K3: a delta-channel graph can putWrites on a checkpoint that an interrupt
// checkpoint then depends on — even without the __interrupt__ marker. A write on a
// real (non-`__`-prefixed) channel must anchor its checkpoint so resume can walk the chain.
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const { config, checkpoint, metadata } = putArgs(threadId);
await saver!.putWrites(
{ configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } },
[['some_delta_channel', { msgs: ['delta'] }]],
'task-1',
);
await saver!.put(config, checkpoint, metadata);
expect(await saver!.getTuple(readConfig(threadId))).toBeDefined();
});
it('does NOT persist an error-only checkpoint OR its write row (failed non-paused turn — no leak)', async () => {
// A turn that throws before any pause records a pending write on the `__error__`
// bookkeeping channel, then a `put` (probe-confirmed against @langchain/langgraph). That
// checkpoint is never HITL-resumable, so the lazy saver must leave NOTHING durable: the
// bookkeeping batch is PARKED (not forwarded) until the checkpoint's fate is known, and the
// discarding `put` drops both — no dead checkpoint, no orphan row in the writes collection.
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const { config, checkpoint, metadata } = putArgs(threadId);
await saver!.putWrites(
{ configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } },
[[ERROR, 'boom']], // '__error__' — bookkeeping channel, not resumable
'task-1',
);
await saver!.put(config, checkpoint, metadata);
expect(await saver!.getTuple(readConfig(threadId))).toBeUndefined();
const db = mongoose.connection.db!;
expect(await db.collection('agent_checkpoints').countDocuments({ thread_id: threadId })).toBe(
0,
);
expect(
await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
).toBe(0);
});
it('preserves bookkeeping writes on a RETAINED checkpoint, in either arrival order', async () => {
// Codex M2 (probe-confirmed): when one Send-sibling interrupts, siblings that completed
// without state updates are recorded as `__no_writes__` pending writes on the SAME retained
// checkpoint. Those markers must persist — dropping them makes LangGraph re-execute the
// completed siblings on resume (duplicated side effects). The saver parks bookkeeping
// batches until the checkpoint is anchored, so both orders must end durable.
const NO_WRITES = '__no_writes__'; // langgraph constants.NO_WRITES (runner marker)
const saver = await getAgentCheckpointer(MONGO_CFG);
const runOrder = async (bookkeepingFirst: boolean) => {
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const { config, checkpoint, metadata } = putArgs(threadId);
const writeCfg = {
configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id },
};
const bookkeeping = () => saver!.putWrites(writeCfg, [[NO_WRITES, null]], 'task-sibling');
const anchor = () => saver!.putWrites(writeCfg, [[INTERRUPT, 'approve?']], 'task-gate');
if (bookkeepingFirst) {
await bookkeeping();
await anchor();
} else {
await anchor();
await bookkeeping();
}
await saver!.put(config, checkpoint, metadata);
const tuple = await saver!.getTuple(readConfig(threadId));
expect(tuple).toBeDefined();
const channels = (tuple?.pendingWrites ?? []).map((w) => w[1]);
expect(channels).toContain(INTERRUPT);
expect(channels).toContain(NO_WRITES);
};
await runOrder(true); // parked, then flushed by the anchoring batch
await runOrder(false); // forwarded directly (checkpoint already anchored)
});
it('un-anchors a checkpoint whose putWrites failed (no phantom pause persisted)', async () => {
// A transient Mongo failure while writing the interrupt batch must not leave a persisted
// checkpoint with MISSING pending writes: LangGraph dispatches the matching `put`
// concurrently with `putWrites` (probe-confirmed), so the failed batch's anchor is removed
// on rejection and that `put` discards the checkpoint instead of saving a phantom pause.
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const { config, checkpoint, metadata } = putArgs(threadId);
const spy = jest
.spyOn(MongoDBSaver.prototype, 'putWrites')
.mockRejectedValueOnce(new Error('transient mongo failure'));
try {
await expect(
saver!.putWrites(
{
configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id },
},
[[INTERRUPT, 'approve?']],
'task-1',
),
).rejects.toThrow('transient mongo failure');
} finally {
spy.mockRestore();
}
// The `put` LangGraph issues for that checkpoint finds no anchor → discarded.
await saver!.put(config, checkpoint, metadata);
expect(await saver!.getTuple(readConfig(threadId))).toBeUndefined();
const count = await mongoose.connection
.db!.collection('agent_checkpoints')
.countDocuments({ thread_id: threadId });
expect(count).toBe(0);
});
it('persists an interrupt checkpoint and carries its __interrupt__ pending write', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
await seedInterruptCheckpoint(saver!, threadId);
const tuple = await saver!.getTuple(readConfig(threadId));
expect(tuple).toBeDefined();
// pendingWrites entries are [taskId, channel, value]; the interrupt is on INTERRUPT.
expect((tuple?.pendingWrites ?? []).some((w) => w[1] === INTERRUPT)).toBe(true);
});
it('end-to-end: a real graph writes nothing on a clean run, a checkpoint on interrupt', async () => {
const { StateGraph, START, END, interrupt, Annotation } = await import('@langchain/langgraph');
const saver = await getAgentCheckpointer(MONGO_CFG);
const coll = mongoose.connection.db!.collection('agent_checkpoints');
const State = Annotation.Root({ x: Annotation });
const build = (withInterrupt: boolean) =>
new StateGraph(State)
.addNode('a', () => {
if (withInterrupt) {
interrupt('approve?');
}
return { x: 'done' };
})
.addEdge(START, 'a')
.addEdge('a', END)
// version skew between checkpoint-mongodb's BaseCheckpointSaver and langgraph's.
.compile({ checkpointer: saver as never });
// Clean run → nothing persisted.
const tClean = `convo-${new mongoose.Types.ObjectId().toString()}`;
await build(false).invoke(
{ x: 'start' },
{ configurable: { thread_id: tClean }, durability: 'exit' },
);
expect(await coll.countDocuments({ thread_id: tClean })).toBe(0);
// Interrupt run → a checkpoint is persisted and the graph reports a pending pause.
const tPause = `convo-${new mongoose.Types.ObjectId().toString()}`;
const pauseGraph = build(true);
await pauseGraph.invoke(
{ x: 'start' },
{ configurable: { thread_id: tPause }, durability: 'exit' },
);
expect(await coll.countDocuments({ thread_id: tPause })).toBeGreaterThan(0);
const state = await pauseGraph.getState({ configurable: { thread_id: tPause } });
expect(state.next.length).toBeGreaterThan(0); // the interrupted node is still pending → resumable
});
it('end-to-end: a real graph that THROWS before pausing persists no checkpoint', async () => {
// F2: a failed non-paused turn records an `__error__` pending write + a put. The lazy
// saver must discard it so a conversation that errors (and is never retried) leaves nothing
// durable behind — the clean-path prune that used to catch this was removed.
const { StateGraph, START, END, Annotation } = await import('@langchain/langgraph');
const saver = await getAgentCheckpointer(MONGO_CFG);
const coll = mongoose.connection.db!.collection('agent_checkpoints');
const State = Annotation.Root({ x: Annotation });
const boomGraph = new StateGraph(State)
.addNode('a', () => {
throw new Error('boom');
})
.addEdge(START, 'a')
.addEdge('a', END)
.compile({ checkpointer: saver as never });
const tErr = `convo-${new mongoose.Types.ObjectId().toString()}`;
await expect(
boomGraph.invoke({ x: 'start' }, { configurable: { thread_id: tErr }, durability: 'exit' }),
).rejects.toThrow('boom');
// Nothing durable: neither the checkpoint nor an orphan row in the writes collection.
expect(await coll.countDocuments({ thread_id: tErr })).toBe(0);
const writesColl = mongoose.connection.db!.collection('agent_checkpoint_writes');
expect(await writesColl.countDocuments({ thread_id: tErr })).toBe(0);
});
it('end-to-end: an interrupt persists, then resumes to completion with the approval value', async () => {
// Guards the `putWrites` change: the `__interrupt__` write must still be forwarded (it is
// resumable) so a paused run rehydrates and the resume value flows in. Mirrors the real HITL
// round-trip across a fresh `invoke` on the same thread_id.
const { StateGraph, START, END, interrupt, Annotation, Command } = await import(
'@langchain/langgraph'
);
const saver = await getAgentCheckpointer(MONGO_CFG);
const State = Annotation.Root({ approved: Annotation });
const graph = new StateGraph(State)
.addNode('gate', () => ({ approved: interrupt('approve?') }))
.addNode('done', () => ({}))
.addEdge(START, 'gate')
.addEdge('gate', 'done')
.addEdge('done', END)
.compile({ checkpointer: saver as never });
const thread = `convo-${new mongoose.Types.ObjectId().toString()}`;
const cfg = { configurable: { thread_id: thread }, durability: 'exit' as const };
// Pause at the interrupt.
await graph.invoke({ approved: null }, cfg);
const paused = await graph.getState(cfg);
expect(paused.next.length).toBeGreaterThan(0);
// Resume with the approval — the paused run rehydrates from the durable interrupt checkpoint.
const out = await graph.invoke(new Command({ resume: 'YES' }), cfg);
expect(out.approved).toBe('YES');
});
it('end-to-end: completed Send-siblings are NOT re-executed after a pause (no duplicate side effects)', async () => {
// Codex M2 end-to-end: a Send fan-out where 'b' pauses for approval while 'a'/'c' complete
// with side effects but NO state writes (→ `__no_writes__` markers on the retained
// checkpoint). On resume — through a REBUILT graph, as resume.js rebuilds the Run — the
// completed siblings must not run again. Before the buffering fix this probe measured
// effects {a:2, c:2}: the dropped markers made LangGraph re-execute both siblings.
const { StateGraph, START, END, interrupt, Annotation, Send, Command } = await import(
'@langchain/langgraph'
);
const saver = await getAgentCheckpointer(MONGO_CFG);
const effects: Record<string, number> = {};
const State = Annotation.Root({
items: Annotation({ reducer: (a: string[], b: string[]) => (a ?? []).concat(b ?? []) }),
results: Annotation({ reducer: (a: string[], b: string[]) => (a ?? []).concat(b ?? []) }),
});
const build = () =>
new StateGraph(State)
.addNode('fan', () => ({}))
.addNode('work', (s: { item?: string }) => {
if (s.item === 'b') {
return { results: [`b:${interrupt('approve b?')}`] };
}
effects[s.item!] = (effects[s.item!] ?? 0) + 1; // side effect, no state update
return {}; // no channel writes → langgraph records a `__no_writes__` marker
})
.addConditionalEdges(
'fan',
(s) => s.items.map((i: string) => new Send('work', { item: i })),
['work'],
)
.addEdge(START, 'fan')
.addEdge('work', END)
.compile({ checkpointer: saver as never });
const thread = `convo-${new mongoose.Types.ObjectId().toString()}`;
const cfg = { configurable: { thread_id: thread }, durability: 'exit' as const };
await build().invoke({ items: ['a', 'b', 'c'], results: [] }, cfg); // pauses on 'b'
expect(effects).toEqual({ a: 1, c: 1 });
// Resume on a REBUILT graph sharing the durable saver (mirrors resume.js).
const out = await build().invoke(new Command({ resume: 'YES' }), cfg);
expect(out.results).toEqual(['b:YES']);
expect(effects).toEqual({ a: 1, c: 1 }); // siblings did NOT re-execute
});
});
describe('LazyMongoSaver checkpoint size guard (mongodb-memory-server integration)', () => {
// Guards the single-document ceiling: a checkpoint embeds the whole message history, so a
// large conversation can serialize past MongoDB's 16 MB limit. The guard measures the
// serialized state on the persist path, WARNS past a soft threshold, and REJECTS past a hard
// limit BEFORE the write — a typed CheckpointTooLargeError instead of a raw BSONObjectTooLarge.
// Thresholds are shrunk here so payloads stay tiny while exercising the real serde + Mongo.
const clientForSaver = () =>
// mongoose vends the live MongoClient; the driver type resolves to a different `mongodb` copy
// than checkpoint-mongodb's, so the cast mirrors buildMongoSaver in checkpointer.ts.
mongoose.connection.getClient() as unknown as ConstructorParameters<
typeof MongoDBSaver
>[0]['client'];
const makeSaver = (overrides?: { warnBytes?: number; hardLimitBytes?: number }) =>
new LazyMongoSaver({
client: clientForSaver(),
checkpointCollectionName: 'agent_checkpoints',
checkpointWritesCollectionName: 'agent_checkpoint_writes',
ttl: 3600,
...overrides,
});
/** Seed an interrupt anchor for a checkpoint whose serialized state is inflated to ~`payloadBytes`. */
async function seedSizedInterrupt(saver: LazyMongoSaver, threadId: string, payloadBytes: number) {
const checkpoint = emptyCheckpoint();
checkpoint.channel_values = { messages: 'x'.repeat(payloadBytes) };
await saver.putWrites(
{ configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } },
[[INTERRUPT, 'approve?']],
'task-1',
);
const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } };
const metadata = { source: 'input' as const, step: -1, writes: null, parents: {} };
return { checkpoint, config, metadata };
}
afterEach(() => {
jest.restoreAllMocks();
});
it('persists a checkpoint under the soft threshold', async () => {
const saver = makeSaver({ warnBytes: 5_000, hardLimitBytes: 50_000 });
await saver.setup();
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const { checkpoint, config, metadata } = await seedSizedInterrupt(saver, threadId, 100);
await saver.put(config, checkpoint, metadata);
expect(await saver.getTuple(readConfig(threadId))).toBeDefined();
});
it('persists but WARNS when a checkpoint crosses the soft threshold', async () => {
const warnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 50_000 });
await saver.setup();
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const { checkpoint, config, metadata } = await seedSizedInterrupt(saver, threadId, 2_000);
await saver.put(config, checkpoint, metadata);
expect(await saver.getTuple(readConfig(threadId))).toBeDefined();
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('soft threshold'));
});
it('REFUSES to persist and throws CheckpointTooLargeError over the hard limit', async () => {
jest.spyOn(logger, 'error').mockImplementation(() => logger);
const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 2_000 });
await saver.setup();
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const { checkpoint, config, metadata } = await seedSizedInterrupt(saver, threadId, 10_000);
await expect(saver.put(config, checkpoint, metadata)).rejects.toBeInstanceOf(
CheckpointTooLargeError,
);
// Nothing durable was written for that thread — getTuple reads the checkpoint document.
expect(await saver.getTuple(readConfig(threadId))).toBeUndefined();
expect(
await mongoose.connection.db!.collection('agent_checkpoints').countDocuments({
thread_id: threadId,
}),
).toBe(0);
});
it('counts metadata toward the ceiling — refuses when the checkpoint is under but metadata pushes over', async () => {
// MongoDBSaver stores the serialized checkpoint AND metadata in one document, so a
// just-under-limit checkpoint with large metadata still overflows. The guard must catch it
// as a typed CheckpointTooLargeError, not let it fall through to a raw BSONObjectTooLarge.
jest.spyOn(logger, 'error').mockImplementation(() => logger);
const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 2_000 });
await saver.setup();
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const checkpoint = emptyCheckpoint();
checkpoint.channel_values = { messages: 'x'.repeat(400) }; // checkpoint alone well UNDER 2 KB
await saver.putWrites(
{ configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } },
[[INTERRUPT, 'approve?']],
'task-1',
);
const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } };
// Metadata alone pushes checkpoint + metadata over the 2 KB hard limit.
const metadata = {
source: 'input' as const,
step: -1,
writes: { payload: 'y'.repeat(4_000) },
parents: {},
};
await expect(saver.put(config, checkpoint, metadata)).rejects.toBeInstanceOf(
CheckpointTooLargeError,
);
expect(await saver.getTuple(readConfig(threadId))).toBeUndefined();
});
it('counts metadata_search (the raw metadata copy) — refuses when serialized-only would pass', async () => {
// MongoDBSaver stores `metadata_search: metadata` — the WHOLE raw metadata a
// SECOND time in the same document. Sized so checkpoint + serialized metadata is
// UNDER the limit but adding the raw metadata_search copy pushes it over: the guard
// must count that copy, else the doc slips the preflight and overflows on the wire.
jest.spyOn(logger, 'error').mockImplementation(() => logger);
const saver = makeSaver({ warnBytes: 500, hardLimitBytes: 6_000 });
await saver.setup();
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const checkpoint = emptyCheckpoint();
checkpoint.channel_values = { messages: 'x'.repeat(200) };
await saver.putWrites(
{ configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } },
[[INTERRUPT, 'approve?']],
'task-1',
);
const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } };
// ~3 KB metadata: checkpoint + serialized metadata (~3.2 KB) is under 6 KB, but the
// raw metadata_search copy (~another 3 KB) pushes checkpoint+metadata+metadata_search
// over 6 KB. The pre-fix guard (checkpoint + serialized metadata only) would pass here.
const metadata = {
source: 'loop' as const,
step: 1,
writes: { payload: 'y'.repeat(3_000) },
parents: {},
};
await expect(saver.put(config, checkpoint, metadata)).rejects.toBeInstanceOf(
CheckpointTooLargeError,
);
expect(await saver.getTuple(readConfig(threadId))).toBeUndefined();
});
it('flushes bookkeeping parked during the size-serialization window (not dropped on resume)', async () => {
// `put` consumes the write anchor, then AWAITS `assertCheckpointFitsDocument` (serialization).
// A bookkeeping-only putWrites dispatched in that window sees no anchor and no persisted
// marker, so it parks — and must be flushed once the checkpoint persists, or a resume
// re-executes the completed sibling.
const NO_WRITES = '__no_writes__';
const saver = makeSaver({ warnBytes: 5_000, hardLimitBytes: 50_000 });
await saver.setup();
const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
const checkpoint = emptyCheckpoint();
const writeCfg = {
configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id },
};
await saver.putWrites(writeCfg, [[INTERRUPT, 'approve?']], 'task-gate');
// Pause the checkpoint serialization inside `put` so the bookkeeping batch lands mid-window.
const serde = (saver as unknown as { serde: { dumpsTyped: (v: unknown) => unknown } }).serde;
const realDumps = serde.dumpsTyped.bind(serde);
let releaseGate!: () => void;
const gate = new Promise<void>((resolve) => {
releaseGate = resolve;
});
let paused = false;
jest.spyOn(serde, 'dumpsTyped').mockImplementation(async (value: unknown) => {
if (!paused) {
paused = true;
await gate; // hold inside assertCheckpointFitsDocument (anchor consumed, not yet persisted)
}
return realDumps(value);
});
const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } };
const metadata = { source: 'input' as const, step: -1, writes: null, parents: {} };
const putPromise = saver.put(config, checkpoint, metadata); // suspends at the gate
await saver.putWrites(writeCfg, [[NO_WRITES, null]], 'task-sibling'); // parks in the window
releaseGate();
await putPromise;
const tuple = await saver.getTuple(readConfig(threadId));
const channels = (tuple?.pendingWrites ?? []).map((w) => w[1]);
expect(channels).toContain(INTERRUPT);
expect(channels).toContain(NO_WRITES); // flushed, not dropped
});
});
describe('setupCheckpointIndexes on a single-index-build engine (mongodb-memory-server)', () => {
const CHECKPOINTS = 'single_build_checkpoints';
const WRITES = 'single_build_checkpoint_writes';
/** Amazon DocumentDB: "Existing index build in progress on the same collection." */
const INDEX_BUILD_ALREADY_IN_PROGRESS = 40333;
const clientForSaver = () =>
mongoose.connection.getClient() as unknown as ConstructorParameters<
typeof MongoDBSaver
>[0]['client'];
/** Bound to Mongoose's database the way `buildMongoSaver` binds it, so the
* index assertions below read the collections the saver actually built. */
const makeSaver = () =>
new LazyMongoSaver({
client: clientForSaver(),
dbName: mongoose.connection.db?.databaseName,
checkpointCollectionName: CHECKPOINTS,
checkpointWritesCollectionName: WRITES,
ttl: 3600,
});
function indexBuildInProgressError(): Error {
return new mongoose.mongo.MongoServerError({
ok: 0,
code: INDEX_BUILD_ALREADY_IN_PROGRESS,
errmsg:
'Existing index build in progress on the same collection. Collection is limited to a single index build at a time.',
});
}
interface IndexBuild {
collectionName: string;
indexName: string;
build: () => Promise<string>;
}
/**
* Routes every `createIndex` through `handler`, naming the index the way the
* server would. Patched on the driver prototype because the saver fetches a
* fresh `Collection` handle per call.
*/
function patchCreateIndex(handler: (build: IndexBuild) => Promise<string>): () => void {
const prototype = mongoose.mongo.Collection.prototype;
const createIndex = prototype.createIndex;
prototype.createIndex = function (this: mongoose.mongo.Collection, spec, options) {
const indexName =
options?.name ??
Object.entries(spec)
.map(([field, direction]) => `${field}_${direction}`)
.join('_');
return handler({
collectionName: this.collectionName,
indexName,
build: () => createIndex.call(this, spec, options),
});
};
return () => {
prototype.createIndex = createIndex;
};
}
/**
* Turns the in-memory MongoDB into a single-index-build engine for the two
* checkpoint collections: a build that arrives while another is in flight on
* the same collection is rejected the way DocumentDB rejects it, instead of
* being serialized the way MongoDB does. Re-creating an index that already
* exists starts no build on either engine, so it always passes through.
*/
function enforceSingleIndexBuild(): { rejected: () => number; restore: () => void } {
const inFlight = new Set<string>();
const built = new Set<string>();
let rejected = 0;
const restore = patchCreateIndex(async ({ collectionName, indexName, build }) => {
const key = `${collectionName}:${indexName}`;
if ((collectionName !== CHECKPOINTS && collectionName !== WRITES) || built.has(key)) {
return build();
}
if (inFlight.has(collectionName)) {
rejected += 1;
throw indexBuildInProgressError();
}
inFlight.add(collectionName);
try {
await new Promise((resolve) => setTimeout(resolve, 5));
const name = await build();
built.add(key);
return name;
} finally {
inFlight.delete(collectionName);
}
});
return { rejected: () => rejected, restore };
}
async function indexNames(collectionName: string): Promise<string[]> {
const indexes = await mongoose.connection.db!.collection(collectionName).indexes();
return indexes.map((index) => index.name ?? '');
}
it('models the engine: a raw setup() loses one build per collection', async () => {
const engine = enforceSingleIndexBuild();
try {
const errors = await makeSaver().setup();
expect(errors.map((error) => (error as { code?: number }).code)).toEqual([
INDEX_BUILD_ALREADY_IN_PROGRESS,
INDEX_BUILD_ALREADY_IN_PROGRESS,
]);
expect(engine.rejected()).toBe(2);
} finally {
engine.restore();
}
});
it('re-runs setup() until every checkpoint index exists', async () => {
const engine = enforceSingleIndexBuild();
try {
await expect(
setupCheckpointIndexes(makeSaver(), { peerBuildPollMs: 1, peerBuildDeadlineMs: 5_000 }),
).resolves.toEqual([]);
expect(engine.rejected()).toBe(2);
expect(await indexNames(CHECKPOINTS)).toEqual(
expect.arrayContaining(['thread_ns_checkpoint_idx', 'upserted_at_1']),
);
expect(await indexNames(WRITES)).toEqual(
expect.arrayContaining(['thread_ns_checkpoint_task_idx', 'upserted_at_1']),
);
} finally {
engine.restore();
}
});
it('returns errors that are not a concurrent build without re-running setup()', async () => {
const restore = patchCreateIndex(async ({ collectionName, build }) => {
if (collectionName !== WRITES) {
return build();
}
throw new mongoose.mongo.MongoServerError({
ok: 0,
code: 67,
errmsg: 'CannotCreateIndex: bad index spec',
});
});
try {
const saver = makeSaver();
const setup = jest.spyOn(saver, 'setup');
const errors = await setupCheckpointIndexes(saver, {
peerBuildPollMs: 1,
peerBuildDeadlineMs: 5_000,
});
expect(errors.map((error) => error.message)).toEqual([
'CannotCreateIndex: bad index spec',
'CannotCreateIndex: bad index spec',
]);
expect(setup).toHaveBeenCalledTimes(1);
} finally {
restore();
}
});
it('proceeds with the conflict reported once the peer-build deadline passes', async () => {
const restore = patchCreateIndex(async ({ collectionName, build }) => {
if (collectionName !== CHECKPOINTS) {
return build();
}
throw indexBuildInProgressError();
});
try {
const errors = await setupCheckpointIndexes(makeSaver(), {
peerBuildPollMs: 1,
peerBuildDeadlineMs: 20,
});
expect(errors.map((error) => (error as { code?: number }).code)).toEqual([
INDEX_BUILD_ALREADY_IN_PROGRESS,
]);
} finally {
restore();
}
});
it('keeps the failures reported beside a conflict that outlasts the deadline', async () => {
const restore = patchCreateIndex(async ({ collectionName }) => {
if (collectionName === CHECKPOINTS) {
throw indexBuildInProgressError();
}
throw new mongoose.mongo.MongoServerError({
ok: 0,
code: 67,
errmsg: 'CannotCreateIndex: bad index spec',
});
});
try {
const errors = await setupCheckpointIndexes(makeSaver(), {
peerBuildPollMs: 1,
peerBuildDeadlineMs: 20,
});
expect(errors.map((error) => (error as { code?: number }).code).sort()).toEqual([
INDEX_BUILD_ALREADY_IN_PROGRESS,
67,
67,
]);
} finally {
restore();
}
});
it('propagates a setup() rejection that is not a build conflict', async () => {
/** The driver validates collection names server-side and `setup()` settles
* every build with `Promise.allSettled`, so there is no rejection path to
* trigger for real; a rejecting stand-in covers the contract that such a
* rejection reaches the caller's in-process fallback instead of being
* reported as an index error. */
const saver = makeSaver();
saver.setup = () => Promise.reject(new Error('client closed'));
await expect(setupCheckpointIndexes(saver, { peerBuildPollMs: 1 })).rejects.toThrow(
'client closed',
);
});
});