Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import type { Cluster } from 'ioredis';
import { InMemoryJobStore } from '../implementations/InMemoryJobStore';
import { RedisJobStore } from '../implementations/RedisJobStore';
jest.mock('~/cache/redisTelemetry', () => ({
RedisUseCases: { GENERATION_STREAM: 'generation_stream' },
instrumentIORedisClient: (client: unknown) => client,
}));
type Deferred<T> = {
promise: Promise<T>;
resolve: (value: T) => void;
};
function createDeferred<T>(): Deferred<T> {
let resolve!: (value: T) => void;
const promise = new Promise<T>((resolvePromise) => {
resolve = resolvePromise;
});
return { promise, resolve };
}
async function waitFor(predicate: () => boolean): Promise<void> {
for (let attempt = 0; attempt < 20; attempt++) {
if (predicate()) {
return;
}
await Promise.resolve();
}
expect(predicate()).toBe(true);
}
function jobHashFromCreationCall(call: unknown[]): Record<string, string> {
const keyCount = Number(call[1]);
// JOB_CREATE_LUA receives thirteen scalar arguments before its HSET pairs.
const fields = call.slice(15 + keyCount);
const hash = Object.fromEntries(
Array.from({ length: fields.length / 2 }, (_, index) => [
String(fields[index * 2]),
String(fields[index * 2 + 1]),
]),
);
return hash;
}
describe('RedisJobStore', () => {
test('marks only the exact provider segment drained', async () => {
const evalDrain = jest.fn().mockResolvedValue(1);
const redis = {
isCluster: true,
eval: evalDrain,
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(
store.markProviderExecutionDrained('stream-provider-drain', 123456, 'segment-a'),
).resolves.toBe(true);
const [script, keyCount, jobKey, createdAt, providerExecutionId] = evalDrain.mock.calls[0];
expect(script).toContain('HGET", KEYS[1], "createdAt"');
expect(script).toContain('HGET", KEYS[1], "providerExecutionId"');
expect(script).toContain('HSET", KEYS[1], "providerDrained", "1"');
expect([keyCount, jobKey, createdAt, providerExecutionId]).toEqual([
1,
'stream:{stream-provider-drain}:job',
'123456',
'segment-a',
]);
});
test('starts only the exact still-running initial provider segment', async () => {
const evalBegin = jest.fn().mockResolvedValue(1);
const redis = {
isCluster: true,
eval: evalBegin,
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(
store.beginProviderExecution('stream-provider-begin', 123456, 'segment-a'),
).resolves.toBe(true);
const [script, keyCount, jobKey, createdAt, providerExecutionId] = evalBegin.mock.calls[0];
expect(script).toContain('HGET", KEYS[1], "status") ~= "running"');
expect(script).toContain('HGET", KEYS[1], "providerDrained") ~= "1"');
expect(script).toContain('HSET", KEYS[1], "providerDrained", "0"');
expect(script).toContain('"providerExecutionStartedId", ARGV[2]');
expect([keyCount, jobKey, createdAt, providerExecutionId]).toEqual([
1,
'stream:{stream-provider-begin}:job',
'123456',
'segment-a',
]);
});
test('guards the atomic status transition with the expected creation epoch', async () => {
const evalTransition = jest.fn().mockResolvedValue(0);
const redis = {
isCluster: true,
eval: evalTransition,
hgetall: jest.fn().mockResolvedValue({}),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(
store.transitionStatus('stream-epoch', {
from: 'running',
to: 'error',
expectCreatedAt: 123456,
}),
).resolves.toBe(false);
const [
script,
keyCount,
jobKey,
sequenceKey,
chunksKey,
runStepsKey,
steersKey,
claimedSteersKey,
parkedSteersKey,
generationEpochKey,
receiptsKey,
receiptOrderKey,
from,
actionId,
createdAt,
] = evalTransition.mock.calls[0];
expect(script).toContain('HGET", KEYS[1], "createdAt"');
expect(script).toContain('redis.call("DEL", KEYS[5], KEYS[6])');
expect(script).toContain('redis.call("SET", KEYS[7]');
expect(script).toContain(
'redis.call("SET", KEYS[8], currentCreatedAt, "EX", ttl + generationEpochGraceTtl)',
);
expect(script.indexOf('local ownerUserId')).toBeLessThan(
script.indexOf('expireScheduleJob(KEYS[1], ttl)'),
);
expect(script).toContain('if currentTtl > ttl then ttl = currentTtl');
expect([
keyCount,
jobKey,
sequenceKey,
chunksKey,
runStepsKey,
steersKey,
claimedSteersKey,
parkedSteersKey,
generationEpochKey,
receiptsKey,
receiptOrderKey,
from,
actionId,
createdAt,
]).toEqual([
10,
'stream:{stream-epoch}:job',
'stream:{stream-epoch}:seq',
'stream:{stream-epoch}:chunks',
'stream:{stream-epoch}:runsteps',
'stream:{stream-epoch}:steers',
'stream:{stream-epoch}:steers-claimed',
'stream:{stream-epoch}:parked',
'stream:{stream-epoch}:generation-epoch',
'stream:{stream-epoch}:steer-receipts',
'stream:{stream-epoch}:steer-receipt-order',
'running',
'',
'123456',
]);
});
test('retains a terminal persistence barrier when completed TTL is zero', async () => {
const evalTransition = jest.fn().mockResolvedValue(0);
const redis = {
isCluster: true,
eval: evalTransition,
hgetall: jest.fn().mockResolvedValue({
streamId: 'stream-terminal-barrier',
userId: 'user-1',
status: 'running',
createdAt: '100',
syncSent: '0',
}),
} as unknown as Cluster;
const store = new RedisJobStore(redis, { completedTtl: 0 });
await expect(
store.transitionStatus('stream-terminal-barrier', {
from: 'running',
to: 'aborted',
expectCreatedAt: 100,
patch: {
completedAt: 200,
terminalPersistencePending: true,
terminalPersistenceStartedAt: 200,
},
}),
).resolves.toBe(false);
// ARGV[5] is the terminal job TTL. The persistence owner/recovery path
// gets five minutes even when ordinary completed records are immediate.
expect(evalTransition.mock.calls.find((call) => call[1] === 10)?.[16]).toBe('300');
});
test('retains the generation epoch beyond the paused job TTL', async () => {
const evalTransition = jest.fn().mockResolvedValue(0);
const redis = {
isCluster: true,
eval: evalTransition,
} as unknown as Cluster;
const store = new RedisJobStore(redis, { requiresActionTtl: 4321 });
await store.transitionStatus('stream-paused-epoch', {
from: 'running',
to: 'requires_action',
expectCreatedAt: 123456,
});
const transitionCall = evalTransition.mock.calls.find((call) => call[1] === 10)!;
expect(transitionCall[0]).toContain(
'redis.call("SET", KEYS[8], currentCreatedAt, "EX", ttl + generationEpochGraceTtl)',
);
expect(transitionCall[16]).toBe('4621');
expect(transitionCall[17]).toBe('0');
expect(transitionCall[21]).toBe('300');
});
test('seeds the guarded epoch when reaping a legacy job without a marker', async () => {
const now = jest.spyOn(Date, 'now').mockReturnValue(100_000);
const evalCommand = jest.fn().mockResolvedValue(1);
const redis = {
isCluster: true,
eval: evalCommand,
hgetall: jest
.fn()
.mockResolvedValueOnce({
streamId: 'legacy-reap',
userId: 'user-1',
status: 'running',
createdAt: '1',
syncSent: '0',
})
.mockResolvedValue({}),
smembers: jest.fn().mockResolvedValueOnce(['legacy-reap']).mockResolvedValueOnce([]),
srem: jest.fn().mockResolvedValue(1),
} as unknown as Cluster;
const store = new RedisJobStore(redis, { runningTtl: 60 });
try {
await expect(store.cleanup()).resolves.toBe(1);
} finally {
now.mockRestore();
}
const reapCall = evalCommand.mock.calls[0];
expect(reapCall[0]).toContain('redis.call("SET", KEYS[7], ARGV[1], "EX", tonumber(ARGV[5]))');
expect(reapCall.slice(1)).toEqual([
9,
'stream:{legacy-reap}:job',
'stream:{legacy-reap}:chunks',
'stream:{legacy-reap}:runsteps',
'stream:{legacy-reap}:steers',
'stream:{legacy-reap}:steers-claimed',
'stream:{legacy-reap}:parked',
'stream:{legacy-reap}:generation-epoch',
'stream:{legacy-reap}:steer-receipts',
'stream:{legacy-reap}:steer-receipt-order',
'1',
'100000',
'60000',
'300',
'300',
]);
});
test('guards steer peeks with the expected creation epoch in one Redis script', async () => {
const queuedSteer = {
steerId: 'steer-1',
text: 'keep me',
userId: 'user-1',
createdAt: 123,
};
const evalPeek = jest.fn().mockResolvedValue([JSON.stringify(queuedSteer)]);
const lrange = jest.fn();
const redis = {
isCluster: true,
eval: evalPeek,
lrange,
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(store.peekSteers('stream-peek', 123)).resolves.toEqual([queuedSteer]);
const [script, keyCount, jobKey, steersKey, expectedCreatedAt] = evalPeek.mock.calls[0];
expect(script).toContain('HGET", KEYS[1], "createdAt"');
expect(script).toContain('LRANGE", KEYS[2], 0, -1');
expect([keyCount, jobKey, steersKey, expectedCreatedAt]).toEqual([
2,
'stream:{stream-peek}:job',
'stream:{stream-peek}:steers',
'123',
]);
expect(lrange).not.toHaveBeenCalled();
});
test('writes initial metadata in the atomic job creation', async () => {
const evalJobCreation = jest
.fn()
.mockImplementation((...args: unknown[]) => ['', '', args[Number(args[1]) + 3]]);
const redis = {
isCluster: true,
eval: evalJobCreation,
hgetall: jest.fn(() => jobHashFromCreationCall(evalJobCreation.mock.calls[0])),
sadd: jest.fn().mockResolvedValue(1),
srem: jest.fn().mockResolvedValue(1),
expire: jest.fn().mockResolvedValue(1),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
const job = await store.createJob('stream-metadata', 'user-1', 'conversation-1', undefined, {
conversationId: 'untrusted-conversation',
userMessage: {
messageId: 'message-1',
parentMessageId: 'parent-1',
},
responseMessageId: 'response-1',
mcpRequestBody: {
messageId: 'response-1',
conversationId: 'overridden-conversation',
parentMessageId: 'response-1',
},
sender: 'Agent',
endpoint: 'agents',
iconURL: 'https://example.com/icon.png',
model: 'test-model',
agent_id: 'agent-1',
compact: true,
isTemporary: false,
retentionExpiresAt: '2030-01-01T00:00:00.000Z',
agentEventDeliveryKey: 'completion-delivery-1',
agentEventInvocationKey: 'original-delivery-1',
agentEventInvocationGenerationCreatedAt: 987654,
agentEventDetachedActionProducerRequired: true,
scheduleId: 'schedule-1',
scheduleMCPCompletion: {
scheduleId: 'original-schedule',
ownerId: 'user-1',
tenantId: null,
agentId: 'original-root',
invocationMode: 'delegated',
},
scheduledFor: '2026-08-17T12:00:00.000Z',
scheduleConfigRevision: 4,
scheduleManual: false,
scheduleOutcome: 'interrupted',
scheduleOutcomeError: 'Schedule deleted',
scheduleMCPFailure: {
server: '',
agentId: 'child',
reason: 'tool_policy_denied',
status: 'mcp_permission_denied',
recovery: 'configure',
automaticReplay: false,
},
preserveForScheduleReconcile: true,
promptTokens: 0,
discoveredTools: [],
authoredSkills: [
{ agentId: 'a', name: 'new-skill', id: '507f1f77bcf86cd799439011', primed: true },
],
contextMeta: {
calibrationRatio: 1.25,
encoding: 'claude',
fading: { v: 1, budgetTokens: 50_000, masked: true },
fadingTiers: [{ agentId: 'agent-123', v: 1, budgetTokens: 50_000, masked: true }],
},
compactionSemanticIndex: {
version: 1,
entries: [
{
type: 'activity_phase',
sourceMessageId: 'assistant-history',
sourceContentIndex: 1,
revision: 1,
status: 'committed',
text: 'Verified the release state',
},
],
},
preemptCapable: true,
steerQuotesExecutionId: 'exec-1',
generationProtocolVersion: 2,
resolvedAskUserQuestions: [
{
request: { question: 'Deploy where?' },
output: 'prod',
toolCallId: 'call-1',
},
{
request: { question: 'Legacy missing?' },
output: 'yes',
contentMissing: true,
},
],
});
/**
* `serializeJob` writes every boolean generically but `deserializeJob` is
* an explicit field map, so a new flag silently vanishes on read unless
* it is added there too. For `preemptCapable` that meant interrupt-steer
* degrading to ordinary steering in every Redis deployment.
*/
expect(job.preemptCapable).toBe(true);
/** The abort paths read this flag from a reloaded job, so it has to
* survive the Redis round trip for a stopped compaction to be stamped. */
expect(job.compact).toBe(true);
expect(job.steerQuotesExecutionId).toBe('exec-1');
expect(job.generationProtocolVersion).toBe(2);
expect(job.checkpointNamespace).toEqual(expect.any(String));
expect(job.checkpointNamespace).not.toBe(String(job.createdAt));
expect(job.resolvedAskUserQuestions).toEqual([
{
request: { question: 'Deploy where?' },
output: 'prod',
toolCallId: 'call-1',
},
{
request: { question: 'Legacy missing?' },
output: 'yes',
contentMissing: true,
},
]);
expect(job).toMatchObject({
streamId: 'stream-metadata',
userId: 'user-1',
conversationId: 'conversation-1',
userMessage: {
messageId: 'message-1',
parentMessageId: 'parent-1',
},
responseMessageId: 'response-1',
agentEventDeliveryKey: 'completion-delivery-1',
agentEventInvocationKey: 'original-delivery-1',
agentEventInvocationGenerationCreatedAt: 987654,
agentEventDetachedActionProducerRequired: true,
mcpRequestBody: {
messageId: 'response-1',
conversationId: 'overridden-conversation',
parentMessageId: 'response-1',
},
sender: 'Agent',
endpoint: 'agents',
iconURL: 'https://example.com/icon.png',
model: 'test-model',
agent_id: 'agent-1',
isTemporary: false,
retentionExpiresAt: '2030-01-01T00:00:00.000Z',
scheduleId: 'schedule-1',
scheduleMCPCompletion: {
scheduleId: 'original-schedule',
ownerId: 'user-1',
tenantId: null,
agentId: 'original-root',
invocationMode: 'delegated',
},
scheduledFor: '2026-08-17T12:00:00.000Z',
scheduleConfigRevision: 4,
scheduleManual: false,
scheduleOutcome: 'interrupted',
scheduleOutcomeError: 'Schedule deleted',
scheduleMCPFailure: {
server: '',
agentId: 'child',
reason: 'tool_policy_denied',
status: 'mcp_permission_denied',
recovery: 'configure',
automaticReplay: false,
},
preserveForScheduleReconcile: true,
promptTokens: 0,
discoveredTools: [],
authoredSkills: [
{ agentId: 'a', name: 'new-skill', id: '507f1f77bcf86cd799439011', primed: true },
],
contextMeta: {
calibrationRatio: 1.25,
encoding: 'claude',
fading: { v: 1, budgetTokens: 50_000, masked: true },
fadingTiers: [{ agentId: 'agent-123', v: 1, budgetTokens: 50_000, masked: true }],
},
compactionSemanticIndex: {
version: 1,
entries: [
expect.objectContaining({
type: 'activity_phase',
text: 'Verified the release state',
}),
],
},
resolvedAskUserQuestions: [
{
request: { question: 'Deploy where?' },
output: 'prod',
toolCallId: 'call-1',
},
{
request: { question: 'Legacy missing?' },
output: 'yes',
contentMissing: true,
},
],
});
const creationArgs = evalJobCreation.mock.calls[0];
const storedFields = jobHashFromCreationCall(creationArgs);
expect(storedFields).toMatchObject({
conversationId: 'conversation-1',
responseMessageId: 'response-1',
mcpRequestBody: JSON.stringify({
messageId: 'response-1',
conversationId: 'overridden-conversation',
parentMessageId: 'response-1',
}),
agent_id: 'agent-1',
isTemporary: '0',
retentionExpiresAt: '2030-01-01T00:00:00.000Z',
scheduleId: 'schedule-1',
scheduledFor: '2026-08-17T12:00:00.000Z',
scheduleConfigRevision: '4',
scheduleManual: '0',
scheduleOutcome: 'interrupted',
scheduleOutcomeError: 'Schedule deleted',
scheduleMCPFailure: JSON.stringify({
server: '',
agentId: 'child',
reason: 'tool_policy_denied',
status: 'mcp_permission_denied',
recovery: 'configure',
automaticReplay: false,
}),
preserveForScheduleReconcile: '1',
promptTokens: '0',
discoveredTools: '[]',
authoredSkills: JSON.stringify([
{ agentId: 'a', name: 'new-skill', id: '507f1f77bcf86cd799439011', primed: true },
]),
resolvedAskUserQuestions: JSON.stringify([
{
request: { question: 'Deploy where?' },
output: 'prod',
toolCallId: 'call-1',
},
{
request: { question: 'Legacy missing?' },
output: 'yes',
contentMissing: true,
},
]),
});
});
test.each([
'null',
'42',
'"answer"',
'{}',
'[]',
'[null]',
'[{"request":"Question?","output":"answer","contentIndex":-1}]',
'[{"request":"Question?","output":"answer","contentMissing":false}]',
])('drops malformed resolved ask-user metadata: %s', async (resolvedAskUserQuestions) => {
const redis = {
isCluster: true,
hgetall: jest.fn().mockResolvedValue({
streamId: 'stream-malformed-answer',
userId: 'user-1',
status: 'running',
createdAt: '100',
resolvedAskUserQuestions,
}),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(store.getJob('stream-malformed-answer')).resolves.toMatchObject({
streamId: 'stream-malformed-answer',
resolvedAskUserQuestions: undefined,
});
});
test('restores detached terminal outbox evidence from Redis', async () => {
const evidence = {
version: 1 as const,
deliveryKey: 'trigger-1',
generationCreatedAt: 100,
taskId: 'task-1',
idempotencyKey: 'a'.repeat(64),
status: 'succeeded' as const,
result: 'accepted',
observedAt: 150,
};
const redis = {
isCluster: true,
hgetall: jest.fn().mockResolvedValue({
streamId: 'stream-detached-terminal',
userId: 'user-1',
status: 'complete',
createdAt: '100',
syncSent: '0',
agentEventDetachedTerminalEvidence: JSON.stringify(evidence),
}),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(store.getJob('stream-detached-terminal')).resolves.toMatchObject({
agentEventDetachedTerminalEvidence: evidence,
});
});
test('atomically resets predecessor state when creating a replacement', async () => {
const evalJobCreation = jest
.fn()
.mockImplementation((...args: unknown[]) => [
'user-1',
'',
args[Number(args[1]) + 3],
'',
'1',
'running',
'conversation-before-replacement',
'1',
'provider-before-replacement',
'0',
]);
const redis = {
isCluster: true,
eval: evalJobCreation,
hgetall: jest.fn(() => jobHashFromCreationCall(evalJobCreation.mock.calls[0])),
sadd: jest.fn().mockResolvedValue(1),
srem: jest.fn().mockResolvedValue(1),
expire: jest.fn().mockResolvedValue(1),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
store.setCollectedUsage('stream-replacement', [{ input_tokens: 10 }]);
const replacement = await store.createJob('stream-replacement', 'user-1');
expect(replacement.replacedJob).toMatchObject({
createdAt: 1,
status: 'running',
conversationId: 'conversation-before-replacement',
providerAbortReady: true,
providerExecutionId: 'provider-before-replacement',
providerDrained: false,
});
const [script, keyCount, ...args] = evalJobCreation.mock.calls[0];
expect(script).toContain(
'local retainedEpochRaw = redis.call("GET", KEYS[7]) local retainedEpoch = tonumber(retainedEpochRaw)',
);
expect(script).toContain('if retainedEpochRaw and not isSafeEpoch(retainedEpoch) then');
expect(script).toContain(
'if previousCreatedAt and previousCreatedAt >= createdAt then createdAt = previousCreatedAt + 1 end',
);
expect(script).not.toContain('"checkpointNamespace", tostring(createdAt)');
expect(script).toContain('redis.call("DEL", KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5])');
expect(script).toContain(
'redis.call("SET", KEYS[7], tostring(createdAt), "EX", ttl + generationEpochGraceTtl)',
);
expect([keyCount, ...args.slice(0, 10)]).toEqual([
10,
'stream:{stream-replacement}:job',
'stream:{stream-replacement}:chunks',
'stream:{stream-replacement}:runsteps',
'stream:{stream-replacement}:steers',
'stream:{stream-replacement}:steers-claimed',
'stream:{stream-replacement}:parked',
'stream:{stream-replacement}:generation-epoch',
'stream:{stream-replacement}:steer-receipts',
'stream:{stream-replacement}:steer-receipt-order',
'stream:{stream-replacement}:job',
]);
expect(args[12]).toBe('300');
expect(args[13]).toBe('300');
expect(store.getCollectedUsage('stream-replacement')).toEqual([]);
});
test('an older overlapping create cannot clear a newer local cache', async () => {
const now = jest.spyOn(Date, 'now').mockReturnValue(100);
const redis = {
isCluster: true,
eval: jest.fn().mockResolvedValue(['user-1', '', '100']),
hgetall: jest.fn().mockResolvedValue({
streamId: 'stream-overlap',
userId: 'user-1',
status: 'running',
createdAt: '101',
syncSent: '0',
}),
sadd: jest.fn().mockResolvedValue(1),
srem: jest.fn().mockResolvedValue(1),
expire: jest.fn().mockResolvedValue(1),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
const newerUsage = [{ input_tokens: 20 }];
store.setCollectedUsage('stream-overlap', newerUsage, 101);
try {
await expect(store.createJob('stream-overlap', 'user-1')).rejects.toThrow(
'Generation job was replaced during creation',
);
expect(store.getCollectedUsage('stream-overlap', 101)).toBe(newerUsage);
expect(store.getCollectedUsage('stream-overlap', 100)).toEqual([]);
} finally {
now.mockRestore();
}
});
test('rejects creation when its durable epoch is already terminal', async () => {
const redis = {
isCluster: true,
eval: jest.fn().mockResolvedValue(['', '', '100']),
hgetall: jest.fn().mockResolvedValue({
streamId: 'stream-terminal-create',
userId: 'user-1',
status: 'complete',
createdAt: '100',
syncSent: '0',
}),
sadd: jest.fn().mockResolvedValue(1),
srem: jest.fn().mockResolvedValue(1),
expire: jest.fn().mockResolvedValue(1),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(store.createJob('stream-terminal-create', 'user-1')).rejects.toThrow(
'Generation job was replaced during creation',
);
});
test('guards local content caches by creation epoch', async () => {
const evalRedis = jest.fn().mockResolvedValue(false);
const redis = {
isCluster: true,
eval: evalRedis,
} as unknown as Cluster;
const store = new RedisJobStore(redis);
const replacementContent = [{ type: 'text', text: 'replacement' }];
const replacementUsage = [{ input_tokens: 20 }];
const replacementGraph = {
getContentParts: () => replacementContent,
getRunSteps: () => [{ id: 'replacement-step' }],
};
const staleGraph = {
getContentParts: () => [{ type: 'text', text: 'predecessor' }],
getRunSteps: () => [{ id: 'predecessor-step' }],
};
store.setContentParts('stream-local-epoch', replacementContent, 101);
store.setCollectedUsage('stream-local-epoch', replacementUsage, 101);
store.setGraph('stream-local-epoch', replacementGraph as never, 101);
store.setContentParts('stream-local-epoch', [{ type: 'text', text: 'predecessor' }], 100);
store.setCollectedUsage('stream-local-epoch', [{ input_tokens: 10 }], 100);
store.setGraph('stream-local-epoch', staleGraph as never, 100);
store.setCollectedUsage('stream-local-epoch', [{ input_tokens: 30 }]);
store.clearContentState('stream-local-epoch', 100);
await expect(store.getContentParts('stream-local-epoch', 101)).resolves.toEqual({
content: replacementContent,
});
expect(store.getCollectedUsage('stream-local-epoch', 101)).toBe(replacementUsage);
await expect(store.getRunSteps('stream-local-epoch', 101)).resolves.toEqual([
{ id: 'replacement-step' },
]);
await expect(store.getContentParts('stream-local-epoch', 100)).resolves.toBeNull();
expect(store.getCollectedUsage('stream-local-epoch', 100)).toEqual([]);
await expect(store.getRunSteps('stream-local-epoch', 100)).resolves.toEqual([]);
store.clearContentState('stream-local-epoch', 101);
await expect(store.getContentParts('stream-local-epoch', 101)).resolves.toBeNull();
expect(store.getCollectedUsage('stream-local-epoch', 101)).toEqual([]);
const chunkRead = evalRedis.mock.calls.find(([script]) => String(script).includes('XRANGE'));
expect(chunkRead).toEqual([
expect.stringContaining('HGET", KEYS[1], "createdAt"'),
2,
'stream:{stream-local-epoch}:job',
'stream:{stream-local-epoch}:chunks',
'100',
]);
const runStepsRead = evalRedis.mock.calls.find(([script]) =>
String(script).includes('GET", KEYS[2]'),
);
expect(runStepsRead).toEqual([
expect.stringContaining('HGET", KEYS[1], "createdAt"'),
2,
'stream:{stream-local-epoch}:job',
'stream:{stream-local-epoch}:runsteps',
'100',
]);
});
test('parallelizes Redis Cluster membership bookkeeping with atomic owner registration', async () => {
const evalResult = createDeferred<number>();
const runningMembership = createDeferred<number>();
const requiresActionRemoval = createDeferred<number>();
const terminalHostActionRemoval = createDeferred<number>();
const detachedTerminalHostActionRemoval = createDeferred<number>();
const ownerRegistration = createDeferred<number>();
const started: string[] = [];
const evalJobCreation = jest.fn((_script: string, keyCount: number) => {
if (keyCount === 1) {
started.push('owner');
return ownerRegistration.promise;
}
started.push('job');
return evalResult.promise;
});
const redis = {
isCluster: true,
eval: evalJobCreation,
sadd: jest.fn((key: string) => {
if (key === 'stream:running') {
started.push('running');
return runningMembership.promise;
}
throw new Error(`Unexpected SADD key: ${key}`);
}),
srem: jest.fn((key: string) => {
if (key === 'stream:requires_action') {
started.push('requires_action');
return requiresActionRemoval.promise;
}
if (key === 'stream:agent_event_detached:terminal_host_action:v1') {
started.push('detached_terminal_host_action');
return detachedTerminalHostActionRemoval.promise;
}
started.push('terminal_host_action');
return terminalHostActionRemoval.promise;
}),
hgetall: jest.fn(() => jobHashFromCreationCall(evalJobCreation.mock.calls[0])),
} as unknown as Cluster;
const store = new RedisJobStore(redis, { userJobsSetTtl: 60 });
let settled = false;
const creating = store.createJob('stream-1', 'user-1').then((job) => {
settled = true;
return job;
});
expect(started).toEqual(['job']);
evalResult.resolve(1);
await waitFor(() => started.length === 6);
expect(started).toEqual([
'job',
'running',
'requires_action',
'terminal_host_action',
'detached_terminal_host_action',
'owner',
]);
expect(settled).toBe(false);
ownerRegistration.resolve(1);
await Promise.resolve();
expect(settled).toBe(false);
runningMembership.resolve(1);
await Promise.resolve();
expect(settled).toBe(false);
terminalHostActionRemoval.resolve(1);
await Promise.resolve();
expect(settled).toBe(false);
detachedTerminalHostActionRemoval.resolve(1);
await Promise.resolve();
expect(settled).toBe(false);
requiresActionRemoval.resolve(1);
await expect(creating).resolves.toMatchObject({
streamId: 'stream-1',
userId: 'user-1',
status: 'running',
});
expect(settled).toBe(true);
});
test('reconciles membership again when the generation changes at the post-write check', async () => {
const postWriteCheckStarted = createDeferred<void>();
const releasePostWriteCheck = createDeferred<void>();
const memberships = new Map<string, Set<string>>([
['stream:running', new Set(['stream-race'])],
['stream:requires_action', new Set()],
['stream:user:{user-old}:jobs', new Set(['stream-race'])],
['stream:user:{user-new}:jobs', new Set()],
]);
let durableHash: Record<string, string> = {
streamId: 'stream-race',
userId: 'user-old',
status: 'running',
createdAt: '100',
syncSent: '0',
};
let reads = 0;
const hgetall = jest.fn(async () => {
reads++;
if (reads === 2) {
postWriteCheckStarted.resolve();
await releasePostWriteCheck.promise;
}
return { ...durableHash };
});
const sadd = jest.fn(async (key: string, streamId: string) => {
let members = memberships.get(key);
if (!members) {
members = new Set();
memberships.set(key, members);
}
members.add(streamId);
return 1;
});
const srem = jest.fn(async (key: string, streamId: string) => {
memberships.get(key)?.delete(streamId);
return 1;
});
const redis = {
isCluster: true,
eval: jest.fn(async (_script: string, keyCount: number, ...args: string[]) => {
if (keyCount === 10) {
durableHash = { ...durableHash, status: 'requires_action' };
} else {
const [ownerKey, streamId] = args;
await sadd(ownerKey, streamId);
}
return 1;
}),
hgetall,
sadd,
srem,
expire: jest.fn().mockResolvedValue(1),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
const transitioning = store.transitionStatus('stream-race', {
from: 'running',
to: 'requires_action',
});
await postWriteCheckStarted.promise;
durableHash = {
streamId: 'stream-race',
userId: 'user-new',
status: 'running',
createdAt: '200',
syncSent: '0',
};
releasePostWriteCheck.resolve();
await expect(transitioning).resolves.toBe(true);
expect(reads).toBe(3);
expect(memberships.get('stream:running')).toContain('stream-race');
expect(memberships.get('stream:requires_action')).not.toContain('stream-race');
expect(memberships.get('stream:user:{user-old}:jobs')).not.toContain('stream-race');
expect(memberships.get('stream:user:{user-new}:jobs')).toContain('stream-race');
});
test('passes expected creation epochs to atomic Redis update and delete scripts', async () => {
const evalCommand = jest.fn().mockResolvedValue(0);
const redis = {
isCluster: true,
eval: evalCommand,
hgetall: jest.fn().mockResolvedValue({
streamId: 'stream-guarded',
userId: 'user-1',
status: 'running',
createdAt: '200',
syncSent: '0',
}),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await store.updateJob('stream-guarded', { error: 'late predecessor error' }, 100);
await expect(store.deleteJob('stream-guarded', 100)).resolves.toBe(false);
const updateCall = evalCommand.mock.calls[0];
expect(updateCall[0]).toContain('HGET", KEYS[1], "createdAt"');
expect(updateCall[6]).toBe('100');
const deleteCall = evalCommand.mock.calls[1];
expect(deleteCall[0]).toContain('HGET", KEYS[1], "createdAt"');
expect(deleteCall.slice(1)).toEqual([
5,
'stream:{stream-guarded}:job',
'stream:{stream-guarded}:chunks',
'stream:{stream-guarded}:runsteps',
'stream:{stream-guarded}:steers',
'stream:{stream-guarded}:steers-claimed',
'100',
'0',
]);
});
test('guards chunk appends with the expected creation epoch inside Redis Lua', async () => {
const evalCommand = jest.fn().mockResolvedValue(0);
const redis = {
isCluster: true,
eval: evalCommand,
} as unknown as Cluster;
const store = new RedisJobStore(redis);
const event = { event: 'on_message_delta', data: { text: 'stale' } };
await store.appendChunk('stream-chunk-guarded', event, 100);
const appendCall = evalCommand.mock.calls[0];
expect(appendCall[0]).toContain('currentCreatedAt ~= ARGV[3]');
expect(appendCall[0]).toContain('currentStatus ~= "running"');
expect(appendCall[0]).toContain('currentStatus ~= "requires_action"');
expect(appendCall[0]).toContain('retainedEpoch ~= currentCreatedAt');
expect(appendCall[0]).toContain('redis.call("EXPIRE", KEYS[8], epochTarget)');
expect(appendCall.slice(1)).toEqual([
8,
'stream:{stream-chunk-guarded}:chunks',
'stream:{stream-chunk-guarded}:job',
'stream:{stream-chunk-guarded}:steer-receipts',
'stream:{stream-chunk-guarded}:steer-receipt-order',
'stream:{stream-chunk-guarded}:steers-claimed',
'stream:{stream-chunk-guarded}:steers',
'stream:{stream-chunk-guarded}:parked',
'stream:{stream-chunk-guarded}:generation-epoch',
JSON.stringify(event),
'1500',
'100',
'',
'',
expect.any(String),
'300',
'300',
]);
});
test('guards run-step saves with the expected epoch and pending host-action window inside Redis Lua', async () => {
const evalCommand = jest.fn().mockResolvedValue(0);
const redis = {
isCluster: true,
eval: evalCommand,
} as unknown as Cluster;
const store = new RedisJobStore(redis);
const runSteps = [{ id: 'step-1', type: 'tool_call' }];
await store.saveRunSteps?.('stream-runstep-guarded', runSteps as never, 100);
const saveCall = evalCommand.mock.calls[0];
expect(saveCall[0]).toContain('currentCreatedAt ~= ARGV[3]');
expect(saveCall[0]).toContain('currentStatus ~= "running"');
expect(saveCall[0]).toContain('currentStatus ~= "requires_action"');
expect(saveCall[0]).toContain('detachedAgentEventTerminalHostActionPending');
expect(saveCall[0]).toContain('and not terminalHostActionPending');
expect(saveCall[0]).toContain(
'currentStatus == "requires_action" or terminalHostActionPending',
);
expect(saveCall.slice(1)).toEqual([
2,
'stream:{stream-runstep-guarded}:runsteps',
'stream:{stream-runstep-guarded}:job',
JSON.stringify(runSteps),
'1500',
'100',
]);
});
test('arms terminal host-action discovery before committing the terminal hash', async () => {
const evalTransition = jest.fn().mockResolvedValue(1);
const sadd = jest.fn().mockResolvedValue(1);
const redis = {
isCluster: true,
eval: evalTransition,
sadd,
srem: jest.fn().mockResolvedValue(1),
expire: jest.fn().mockResolvedValue(1),
hgetall: jest
.fn()
.mockResolvedValueOnce({
streamId: 'stream-host-action-prearm',
userId: 'user-1',
status: 'running',
createdAt: '100',
syncSent: '0',
})
.mockResolvedValue({
streamId: 'stream-host-action-prearm',
userId: 'user-1',
status: 'complete',
createdAt: '100',
completedAt: '200',
terminalHostActionPending: '1',
syncSent: '0',
}),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(
store.transitionStatus('stream-host-action-prearm', {
from: 'running',
to: 'complete',
expectCreatedAt: 100,
patch: { completedAt: 200, terminalHostActionPending: true },
}),
).resolves.toBe(true);
expect(sadd.mock.calls[0]).toEqual([
'stream:terminal_host_action',
'["stream-host-action-prearm",100]',
]);
expect(sadd.mock.invocationCallOrder[0]).toBeLessThan(
evalTransition.mock.invocationCallOrder[0],
);
const transitionCall = evalTransition.mock.calls.find((call) => call[1] === 10)!;
expect(transitionCall[16]).toBe('86400');
expect(transitionCall[18]).toBe('86400');
expect(transitionCall[19]).toBe('86400');
});
test('pre-arms detached completion recovery outside the legacy terminal index', async () => {
const evalTransition = jest.fn().mockResolvedValue(1);
const sadd = jest.fn().mockResolvedValue(1);
const detachedIdentity = {
agentEventDeliveryKey: 'completion-delivery',
agentEventInvocationKey: 'original-invocation',
agentEventInvocationGenerationCreatedAt: '90',
};
const redis = {
isCluster: true,
eval: evalTransition,
sadd,
srem: jest.fn().mockResolvedValue(1),
expire: jest.fn().mockResolvedValue(1),
hgetall: jest
.fn()
.mockResolvedValueOnce({
streamId: 'stream-detached-host-action',
userId: 'user-1',
status: 'running',
createdAt: '100',
syncSent: '0',
...detachedIdentity,
})
.mockResolvedValue({
streamId: 'stream-detached-host-action',
userId: 'user-1',
status: 'complete',
createdAt: '100',
completedAt: '200',
detachedAgentEventTerminalHostActionPending: '1',
syncSent: '0',
...detachedIdentity,
}),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(
store.transitionStatus('stream-detached-host-action', {
from: 'running',
to: 'complete',
expectCreatedAt: 100,
patch: { completedAt: 200, terminalHostActionPending: true },
}),
).resolves.toBe(true);
expect(sadd.mock.calls[0]).toEqual([
'stream:agent_event_detached:terminal_host_action:v1',
'["stream-detached-host-action",100]',
]);
expect(sadd.mock.calls).not.toContainEqual([
'stream:terminal_host_action',
'["stream-detached-host-action",100]',
]);
const transitionCall = evalTransition.mock.calls.find((call) => call[1] === 10)!;
expect(transitionCall).toContain('status');
expect(transitionCall).toContain('detached_terminal_pending_v1');
expect(transitionCall).toContain('detachedAgentEventTerminalStatus');
expect(transitionCall).toContain('complete');
});
test('retains an exact detached recovery hint until its terminal CAS is visible', async () => {
const member = '["stream-detached-prearm-race",100]';
const detachedIdentity = {
agentEventDeliveryKey: 'completion-delivery',
agentEventInvocationKey: 'original-invocation',
agentEventInvocationGenerationCreatedAt: '90',
};
const srem = jest.fn().mockResolvedValue(1);
const redis = {
isCluster: true,
eval: jest.fn().mockResolvedValue(1),
smembers: jest.fn().mockResolvedValue([member]),
srem,
sadd: jest.fn().mockResolvedValue(1),
expire: jest.fn().mockResolvedValue(1),
hgetall: jest
.fn()
.mockResolvedValueOnce({
streamId: 'stream-detached-prearm-race',
userId: 'user-1',
status: 'running',
createdAt: '100',
syncSent: '0',
...detachedIdentity,
})
.mockResolvedValueOnce({
streamId: 'stream-detached-prearm-race',
userId: 'user-1',
status: 'detached_terminal_pending_v1',
detachedAgentEventTerminalStatus: 'complete',
detachedAgentEventTerminalHostActionPending: '1',
createdAt: '100',
completedAt: '200',
syncSent: '0',
...detachedIdentity,
}),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(store.getDetachedAgentEventTerminalHostActionJobs()).resolves.toEqual([]);
expect(srem).not.toHaveBeenCalledWith(
'stream:agent_event_detached:terminal_host_action:v1',
member,
);
await expect(store.getDetachedAgentEventTerminalHostActionJobs()).resolves.toEqual([
expect.objectContaining({
streamId: 'stream-detached-prearm-race',
terminalHostActionPending: true,
}),
]);
});
test('acknowledges evidence TTLs without removing a successor retry member', async () => {
const evalClear = jest.fn().mockResolvedValue(1);
const srem = jest.fn().mockResolvedValue(1);
const redis = {
isCluster: true,
eval: evalClear,
srem,
hgetall: jest.fn().mockResolvedValue({}),
} as unknown as Cluster;
const store = new RedisJobStore(redis, {
completedTtl: 300,
chunksAfterCompleteTtl: 7,
runStepsAfterCompleteTtl: 11,
});
await store.clearTerminalHostAction('stream-host-action-clear', 100);
expect(evalClear).toHaveBeenCalledWith(
expect.stringContaining('expireScheduleJob(KEYS[1], tonumber(ARGV[2]))'),
3,
'stream:{stream-host-action-clear}:job',
'stream:{stream-host-action-clear}:chunks',
'stream:{stream-host-action-clear}:runsteps',
'100',
'300',
'7',
'11',
);
expect(evalClear.mock.calls[0][0]).toContain(
'redis.call("HSET", KEYS[1], "status", detachedStatus)',
);
expect(srem).toHaveBeenCalledWith(
'stream:terminal_host_action',
'["stream-host-action-clear",100]',
);
expect(srem).not.toHaveBeenCalledWith(
'stream:terminal_host_action',
'["stream-host-action-clear",101]',
);
});
test('guards asynchronous content cleanup against a replacement epoch', async () => {
const evalCommand = jest.fn().mockResolvedValue(0);
const redis = {
isCluster: true,
eval: evalCommand,
} as unknown as Cluster;
const store = new RedisJobStore(redis);
store.clearContentState('stream-content-guarded', 100);
await waitFor(() => evalCommand.mock.calls.length === 1);
const clearCall = evalCommand.mock.calls[0];
expect(clearCall[0]).toContain('redis.call("EXISTS", KEYS[3]) == 1');
expect(clearCall.slice(1)).toEqual([
3,
'stream:{stream-content-guarded}:chunks',
'stream:{stream-content-guarded}:runsteps',
'stream:{stream-content-guarded}:job',
'100',
]);
});
test('in-memory update and delete guards preserve a replacement epoch', async () => {
const now = jest.spyOn(Date, 'now');
try {
const store = new InMemoryJobStore();
now.mockReturnValue(100);
const original = await store.createJob('stream-memory-guard', 'user-1');
now.mockReturnValue(200);
const replacement = await store.createJob('stream-memory-guard', 'user-1');
await store.updateJob(
'stream-memory-guard',
{ error: 'late predecessor error' },
original.createdAt,
);
await expect(store.deleteJob('stream-memory-guard', original.createdAt)).resolves.toBe(false);
await expect(store.getJob('stream-memory-guard')).resolves.toMatchObject({
createdAt: replacement.createdAt,
status: 'running',
});
await expect(store.deleteJob('stream-memory-guard', replacement.createdAt)).resolves.toBe(
true,
);
} finally {
now.mockRestore();
}
});
test('in-memory replacement advances the creation epoch when the clock does not', async () => {
const now = jest.spyOn(Date, 'now').mockReturnValue(100);
try {
const store = new InMemoryJobStore({ maxJobs: 1 });
const original = await store.createJob('stream-memory-collision', 'user-1');
const replacement = await store.createJob('stream-memory-collision', 'user-1');
expect(original.createdAt).toBe(100);
expect(replacement.createdAt).toBe(101);
await expect(store.getJob('stream-memory-collision')).resolves.toMatchObject({
createdAt: 101,
});
} finally {
now.mockRestore();
}
});
test('in-memory replacement clears predecessor content state', async () => {
const store = new InMemoryJobStore();
await store.createJob('stream-memory-content', 'user-1');
store.setContentParts('stream-memory-content', [{ type: 'text', text: 'predecessor' }]);
store.setCollectedUsage('stream-memory-content', [{ input_tokens: 10 }]);
await store.createJob('stream-memory-content', 'user-1');
await expect(store.getContentParts('stream-memory-content')).resolves.toBeNull();
expect(store.getCollectedUsage('stream-memory-content')).toEqual([]);
});
});
it('routes schedule cleanup reads across Cluster slots without unsafe cross-node pipelines', async () => {
const redis = {
isCluster: true,
sscan: jest.fn().mockResolvedValue(['0', ['["first",1]', '["second",2]']]),
hmget: jest
.fn()
.mockResolvedValueOnce(['1', 'schedule', 'foreign', '1', '0', '1'])
.mockResolvedValueOnce(['2', 'schedule', 'owner', '1', '0', '1']),
pipeline: jest.fn(),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(store.hasScheduleCleanupObligation({ userId: 'owner' })).resolves.toBe(true);
expect(redis.pipeline).not.toHaveBeenCalled();
expect(redis.hmget).toHaveBeenCalledTimes(2);
});
it('does not treat a stale cleanup hint as a successor obligation or a failed read as absence', async () => {
const redis = {
isCluster: true,
sscan: jest.fn().mockResolvedValue(['0', ['["stream",1]']]),
hmget: jest.fn().mockResolvedValue(['2', 'schedule', 'owner', '1', '0', '1']),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(store.hasScheduleCleanupObligation({ scheduleId: 'schedule' })).resolves.toBe(false);
jest.mocked(redis.hmget).mockRejectedValue(new Error('Unavailable'));
await expect(store.hasScheduleCleanupObligation({ userId: 'owner' })).rejects.toThrow(
'Unavailable',
);
});
it('confirms initial retention in Cluster membership before returning a created generation', async () => {
const create = jest
.fn()
.mockImplementation((...args: unknown[]) => ['', '', args[Number(args[1]) + 3]]);
const redis = {
isCluster: true,
eval: create,
hgetall: jest.fn(() => jobHashFromCreationCall(create.mock.calls[0])),
sadd: jest.fn().mockResolvedValue(1),
srem: jest.fn().mockResolvedValue(1),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
const job = await store.createJob('initial-cluster', 'owner', 'initial-cluster', undefined, {
preserveForScheduleReconcile: true,
});
expect(redis.sadd).toHaveBeenCalledWith(
'stream:schedule_reconcile:v1',
JSON.stringify([job.streamId, job.createdAt]),
);
expect(create.mock.calls[0][0]).toContain('expireScheduleJob(KEYS[1], ttl)');
});
it('withholds a newly retained provider when its recovery index cannot be confirmed', async () => {
const create = jest
.fn()
.mockImplementation((...args: unknown[]) => ['', '', args[Number(args[1]) + 3]]);
const redis = {
isCluster: true,
eval: create,
hgetall: jest.fn(() => jobHashFromCreationCall(create.mock.calls[0])),
sadd: jest.fn(async (key: string) => {
if (key === 'stream:schedule_reconcile:v1') throw new Error('Index unavailable');
return 1;
}),
srem: jest.fn().mockResolvedValue(1),
} as unknown as Cluster;
const store = new RedisJobStore(redis);
await expect(
store.createJob('unexposed-retention', 'owner', undefined, undefined, {
preserveForScheduleReconcile: true,
}),
).rejects.toThrow('Created job membership could not be verified');
});
describe('retained creation read repair', () => {
const hash = {
streamId: 'unindexed',
userId: 'owner',
createdAt: '100',
status: 'running',
generationProtocolVersion: '2',
preserveForScheduleReconcile: '1',
providerDrained: '1',
};
const client = (read: jest.Mock, add = jest.fn().mockResolvedValue(1)) => ({
isCluster: true,
hgetall: read,
sadd: add,
srem: jest.fn().mockResolvedValue(1),
eval: jest.fn().mockResolvedValue(1),
});
it('repairs a legacy orphan and stamps its exact epoch after confirming membership', async () => {
const redis = client(jest.fn().mockResolvedValue(hash));
const store = new RedisJobStore(redis as unknown as Cluster);
await expect(store.getJob('unindexed')).resolves.toMatchObject({
createdAt: 100,
status: 'running',
});
expect(redis.sadd).toHaveBeenCalledWith(
'stream:schedule_reconcile:v1',
JSON.stringify(['unindexed', 100]),
);
const commit = redis.eval.mock.calls.find((call) =>
String(call[0]).includes('__scheduleMembershipEpoch'),
);
expect(commit?.slice(1)).toEqual([1, 'stream:{unindexed}:job', '100', '']);
expect(commit?.[0]).toContain('"createdAt") ~= ARGV[1]');
});
it('keeps an unavailable index retryable without marking discovery confirmed', async () => {
const redis = client(
jest.fn().mockResolvedValue(hash),
jest.fn().mockRejectedValue(new Error('Index unavailable')),
);
const store = new RedisJobStore(redis as unknown as Cluster);
await expect(store.getJob('unindexed')).rejects.toThrow(
'Retained creation membership recovery failed',
);
expect(
redis.eval.mock.calls.some((call) => String(call[0]).includes('__scheduleMembershipEpoch')),
).toBe(false);
});
it('repairs and stamps the successor rather than a replaced snapshot', async () => {
const newer = { ...hash, createdAt: '200', userId: 'successor' };
const redis = client(jest.fn().mockResolvedValueOnce(hash).mockResolvedValue(newer));
const store = new RedisJobStore(redis as unknown as Cluster);
await expect(store.getJob('unindexed')).resolves.toMatchObject({
createdAt: 200,
userId: 'successor',
});
expect(redis.sadd).toHaveBeenCalledWith(
'stream:schedule_reconcile:v1',
JSON.stringify(['unindexed', 200]),
);
const commits = redis.eval.mock.calls.filter((call) =>
String(call[0]).includes('__scheduleMembershipEpoch'),
);
expect(commits).toHaveLength(1);
expect(commits[0]?.slice(1)).toEqual([1, 'stream:{unindexed}:job', '200', '']);
});
it('does not confirm a newer same-epoch re-arm with an older repair snapshot', async () => {
const older = { ...hash, __scheduleMembershipRevision: '2' };
const redis = client(jest.fn().mockResolvedValue(older));
const store = new RedisJobStore(redis as unknown as Cluster);
await store.getJob('unindexed');
const commit = redis.eval.mock.calls.find((call) =>
String(call[0]).includes('__scheduleMembershipEpoch'),
);
expect(commit?.slice(1)).toEqual([1, 'stream:{unindexed}:job', '100', '2']);
expect(commit?.[0]).toContain('"__scheduleMembershipRevision") or "") ~= ARGV[2]');
});
it('adds no index round trip to already-confirmed retained reads', async () => {
const redis = client(
jest.fn().mockResolvedValue({ ...hash, __scheduleMembershipEpoch: '100' }),
);
const store = new RedisJobStore(redis as unknown as Cluster);
await expect(store.getJob('unindexed')).resolves.toMatchObject({ createdAt: 100 });
expect(redis.hgetall).toHaveBeenCalledTimes(1);
expect(redis.sadd).not.toHaveBeenCalled();
expect(redis.eval).not.toHaveBeenCalled();
});
});