Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

18.3 KB
Raw Blame
import IoRedis from 'ioredis';
import mongoose from 'mongoose';
import { randomUUID } from 'node:crypto';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { Constants, EModelEndpoint } from 'librechat-data-provider';
import { createMethods, createModels, tenantStorage } from '@librechat/data-schemas';
import type {
  SubagentTaskRuntime,
  SubagentTaskStartRequest,
  SubagentTaskStartResult,
} from '@librechat/agents';
import type { AllMethods } from '@librechat/data-schemas';
import type { AgentTriggerDeliveryPersistence, AgentTriggerService } from './triggers/service';
import type { SubagentTaskWakeupRegistration } from './subagentThreads';
import type { AgentTriggerFetch } from './triggers/host';
import {
  createSubagentCompletionWakeupHandler,
  createSubagentCompletionWakeupResolver,
} from './subagentCompletionWakeup';
import {
  RedisSubagentTaskControlTransport,
  SubagentTaskOwnerUnavailableError,
} from './subagentTaskRouting';
import { buildSubagentThreadTaskConfig, SubagentThreadTaskStore } from './subagentThreads';
import { snapshotActivity, snapshotActivitySummary } from './digest';
import { __resetShutdownStateForTests } from '../app/shutdown';
import { createAgentTriggerService } from './triggers/service';
import { SubagentActivityStream } from './subagentActivity';
import { RedisEventTransport } from '~/stream';

const DB_SETUP_TIMEOUT_MS = 60_000;
const REDIS_URI = process.env.REDIS_URI;
const describeWithRedis = REDIS_URI == null ? describe.skip : describe;

let mongod: MongoMemoryServer;
let methods: AllMethods;
let triggerService: AgentTriggerService | undefined;
const redisClients: IoRedis[] = [];
const taskStores: SubagentThreadTaskStore[] = [];

function accepted(
  started: SubagentTaskStartResult,
): Extract<SubagentTaskStartResult, { accepted: true }> {
  if (!started.accepted) {
    throw new Error('Expected the child task to be accepted.');
  }
  return started;
}

async function waitUntil<T>(read: () => T | undefined | Promise<T | undefined>): Promise<T> {
  const deadline = Date.now() + 10_000;
  while (Date.now() < deadline) {
    const value = await read();
    if (value !== undefined) {
      return value;
    }
    await new Promise<void>((resolve) => setTimeout(resolve, 10));
  }
  throw new Error('Timed out waiting for the cross-replica integration condition.');
}

function redisClient(): IoRedis {
  const client = new IoRedis(REDIS_URI!, {
    enableOfflineQueue: false,
    lazyConnect: true,
    maxRetriesPerRequest: 1,
  });
  redisClients.push(client);
  return client;
}

async function connectedRedisClient(): Promise<IoRedis> {
  const client = redisClient();
  await client.connect();
  return client;
}

async function createRoutingTransport(instanceId: string, namespace: string) {
  const [publisher, subscriber] = await Promise.all([
    connectedRedisClient(),
    connectedRedisClient(),
  ]);
  return new RedisSubagentTaskControlTransport(publisher, subscriber, {
    instanceId,
    namespace,
    registrationHeartbeatMs: 1_000,
    requestTimeoutMs: 1_000,
    retryDelayMs: 25,
  });
}

async function createActivityStream(): Promise<SubagentActivityStream> {
  const [publisher, subscriber] = await Promise.all([
    connectedRedisClient(),
    connectedRedisClient(),
  ]);
  return new SubagentActivityStream(new RedisEventTransport(publisher, subscriber));
}

function taskRequest(
  scopeId: string,
  input: string,
  subagentType: string,
  run: SubagentTaskStartRequest['run'],
): SubagentTaskStartRequest {
  return {
    scopeId,
    idempotencyKey: randomUUID(),
    parentRunId: 'parent-response',
    parentAgentId: 'agent_parent',
    parentToolCallId: randomUUID(),
    input,
    subagentKind: 'agent',
    subagentType,
    run,
  };
}

async function saveParent(userId: string, tenantId: string, conversationId: string): Promise<void> {
  await tenantStorage.run({ tenantId, userId }, async () => {
    await methods.saveConvo(
      { userId },
      {
        conversationId,
        tenantId,
        endpoint: EModelEndpoint.agents,
        title: 'Cross-replica parent',
        agent_id: 'agent_parent',
      },
    );
    await methods.saveMessage(
      { userId },
      {
        messageId: 'parent-response',
        conversationId,
        parentMessageId: String(Constants.NO_PARENT),
        sender: 'Director',
        text: 'I dispatched two child tasks.',
        endpoint: EModelEndpoint.agents,
        isCreatedByUser: false,
      },
    );
  });
}

beforeAll(async () => {
  mongod = await MongoMemoryServer.create();
  createModels(mongoose);
  methods = createMethods(mongoose);
  await mongoose.connect(mongod.getUri(), { autoIndex: false });
}, DB_SETUP_TIMEOUT_MS);

afterEach(async () => {
  await triggerService?.stop();
  triggerService = undefined;
  __resetShutdownStateForTests();
  await Promise.all(
    taskStores.splice(0).map(async (store) => {
      await store.destroyTaskControlTransport().catch(() => undefined);
      store.destroyActivityStream();
    }),
  );
  await Promise.all(redisClients.splice(0).map((client) => client.quit().catch(() => undefined)));
  await mongoose.connection.db?.dropDatabase();
});

afterAll(async () => {
  await mongoose.disconnect();
  await mongod.stop();
}, DB_SETUP_TIMEOUT_MS);

describeWithRedis('subagent cross-replica orchestration', () => {
  it('routes controls to the execution owner and delivers two sibling wakeups once after owner loss', async () => {
    const userId = new mongoose.Types.ObjectId().toString();
    const tenantId = 'tenant-cross-replica';
    const parentConversationId = randomUUID();
    await saveParent(userId, tenantId, parentConversationId);

    const continuationEffects = new Map<string, { input: string; parentMessageId: string }>();
    let dropFirstReceipt = true;
    const fetcher = jest.fn<ReturnType<AgentTriggerFetch>, Parameters<AgentTriggerFetch>>(
      async (_input, init) => {
        const body = JSON.parse(String(init?.body)) as {
          text: string;
          parentMessageId: string;
          conversationId: string;
          clientRequestId: string;
        };
        if (!continuationEffects.has(body.clientRequestId)) {
          const userMessageId = `${body.clientRequestId}:user`;
          const assistantMessageId = `${body.clientRequestId}:assistant`;
          await tenantStorage.run({ tenantId, userId }, async () => {
            await methods.saveMessage(
              { userId },
              {
                messageId: userMessageId,
                conversationId: body.conversationId,
                parentMessageId: body.parentMessageId,
                sender: 'User',
                text: body.text,
                endpoint: EModelEndpoint.agents,
                isCreatedByUser: true,
              },
            );
            await methods.saveMessage(
              { userId },
              {
                messageId: assistantMessageId,
                conversationId: body.conversationId,
                parentMessageId: userMessageId,
                sender: 'Director',
                text: 'Accepted the child result.',
                endpoint: EModelEndpoint.agents,
                isCreatedByUser: false,
              },
            );
          });
          continuationEffects.set(body.clientRequestId, {
            input: body.text,
            parentMessageId: body.parentMessageId,
          });
          /** The parent continuation committed, but its HTTP receipt was lost. The
           * durable delivery must retry with the same client request identity. */
          if (dropFirstReceipt) {
            dropFirstReceipt = false;
            throw new Error('response lost after admission');
          }
        }
        return new Response(
          JSON.stringify({
            status: 'started',
            streamId: body.conversationId,
            conversationId: body.conversationId,
            generationCreatedAt: Date.now(),
          }),
          { status: 200, headers: { 'content-type': 'application/json' } },
        );
      },
    );

    const resolver = createSubagentCompletionWakeupResolver({
      methods,
      getGenerationJob: async () => null,
    });
    triggerService = createAgentTriggerService({
      methods: methods as AllMethods & AgentTriggerDeliveryPersistence,
      fetch: fetcher,
      mintToken: () => 'trigger-token',
      prepareContinue: resolver,
      deliveryOptions: { concurrency: 2, tickMs: 1_000, retryBaseMs: 5, retryCapMs: 20 },
    });
    await triggerService.initialize({
      address: { address: '127.0.0.1', family: 'IPv4', port: 3080 },
    });

    const receipts: Array<{ deliveryKey: string; replayed: boolean }> = [];
    const registrations: SubagentTaskWakeupRegistration[] = [];
    const wakeup = createSubagentCompletionWakeupHandler(async (envelope, options) => {
      const receipt = await triggerService!.enqueue(envelope, options);
      receipts.push(receipt);
      return receipt;
    });
    const ownerStore = new SubagentThreadTaskStore(methods, {
      onTaskPrepared: async (registration) => {
        registrations.push(registration);
        /** A crash-retried preparation can invoke the producer again; enqueueing the
         * same task identity must retain one durable delivery. */
        await wakeup(registration);
        await wakeup(registration);
      },
    });
    const requesterStore = new SubagentThreadTaskStore(methods);
    taskStores.push(ownerStore, requesterStore);
    const namespace = `subagent-integration-${randomUUID()}`;
    await ownerStore.configureTaskControlTransport(
      await createRoutingTransport('execution-owner', namespace),
    );
    await requesterStore.configureTaskControlTransport(
      await createRoutingTransport('delivery-owner', namespace),
    );
    ownerStore.configureActivityStream(await createActivityStream());
    requesterStore.configureActivityStream(await createActivityStream());
    const config = buildSubagentThreadTaskConfig(
      ownerStore,
      { userId, tenantId, parentConversationId },
      { completionWakeups: true },
    );

    const releases: Array<() => void> = [];
    const entered: Promise<void>[] = [];
    const childRun = (result: string) => {
      let markEntered = (): void => undefined;
      entered.push(new Promise<void>((resolve) => (markEntered = resolve)));
      return async (runtime: SubagentTaskRuntime) => {
        markEntered();
        return new Promise<{ content: string }>((resolve) =>
          releases.push(() => {
            runtime.reportProgress({
              runId: 'root-run',
              parentRunId: 'parent-run',
              subagentRunId: runtime.taskId,
              subagentType: 'worker',
              subagentKind: 'agent',
              subagentAgentId: 'agent-worker',
              parentToolCallId: 'tool-call',
              depth: 1,
              ancestry: [],
              phase: 'message_delta',
              data: { delta: { content: [{ type: 'text', text: result }] } },
              timestamp: new Date().toISOString(),
            });
            resolve({ content: result });
          }),
        );
      };
    };
    const first = config.store.start(
      taskRequest(
        config.scopeId,
        'first child task',
        'researcher',
        childRun('First durable result.'),
      ),
    );
    const second = config.store.start(
      taskRequest(
        config.scopeId,
        'second child task',
        'reviewer',
        childRun('Second durable result.'),
      ),
    );
    await Promise.all(entered);

    const firstTaskId = accepted(first).task.taskId;
    const secondTaskId = accepted(second).task.taskId;
    const remoteActivity: unknown[] = [];
    let resolveRemoteDone!: (status: string) => void;
    const remoteDone = new Promise<string>((resolve) => {
      resolveRemoteDone = resolve;
    });
    const remoteSubscription = requesterStore.subscribeActivity(
      accepted(first).task.threadId!,
      firstTaskId,
      {
        onEvent: (event) =>
          remoteActivity.push(
            ...(event.event === 'subagent_activity_replay' ? event.data : [event]),
          ),
        onDone: (event) => resolveRemoteDone(event.status),
      },
    );
    await remoteSubscription.ready;
    await expect(requesterStore.listTasks(config.scopeId)).resolves.toEqual(
      expect.arrayContaining([
        expect.objectContaining({ taskId: firstTaskId, status: 'running' }),
        expect.objectContaining({ taskId: secondTaskId, status: 'running' }),
      ]),
    );
    await expect(
      requesterStore.controlTask(
        config.scopeId,
        firstTaskId,
        { action: 'queue', message: 'Include the primary source.' },
        'cross-replica-steer',
      ),
    ).resolves.toMatchObject({ status: 'accepted' });

    releases.forEach((release) => release());
    await expect(remoteDone).resolves.toBe('completed');
    expect(remoteActivity).toEqual([
      expect.objectContaining({
        event: 'on_subagent_update',
        data: expect.objectContaining({ subagentRunId: firstTaskId }),
      }),
    ]);
    await waitUntil(() => {
      const tasks = [
        ownerStore.get(config.scopeId, firstTaskId),
        ownerStore.get(config.scopeId, secondTaskId),
      ];
      return tasks.every((task) => task?.status === 'completed') ? true : undefined;
    });
    await ownerStore.destroyTaskControlTransport();

    await expect(
      requesterStore.controlTask(
        config.scopeId,
        firstTaskId,
        { action: 'cancel' },
        'owner-gone-control',
      ),
    ).rejects.toBeInstanceOf(SubagentTaskOwnerUnavailableError);

    const uniqueReceipts = new Map(receipts.map((receipt) => [receipt.deliveryKey, receipt]));
    expect(registrations).toHaveLength(2);
    expect(receipts).toHaveLength(4);
    expect(uniqueReceipts.size).toBe(2);
    expect(receipts.filter((receipt) => receipt.replayed)).toHaveLength(2);

    const deliveries = await waitUntil(async () => {
      const deliveries = await Promise.all(
        [...uniqueReceipts.keys()].map((key) => triggerService!.getDelivery(key)),
      );
      const dead = deliveries.find((delivery) => delivery?.status === 'dead');
      if (dead != null) {
        throw new Error(`Wakeup dead-lettered: ${JSON.stringify(dead.lastError)}`);
      }
      return deliveries.every((delivery) => delivery?.status === 'succeeded')
        ? deliveries
        : undefined;
    });

    expect(continuationEffects.size).toBe(2);
    expect(fetcher).toHaveBeenCalledTimes(3);
    expect(deliveries.map((delivery) => delivery?.attempts).sort()).toEqual([1, 2]);
    expect([...continuationEffects.values()].map(({ input }) => input)).toEqual(
      expect.arrayContaining([
        expect.stringContaining('First durable result.'),
        expect.stringContaining('Second durable result.'),
      ]),
    );
    expect([...continuationEffects.values()].map(({ parentMessageId }) => parentMessageId)).toEqual(
      expect.arrayContaining([
        'parent-response',
        expect.stringMatching(/^trigger_[a-f0-9]{64}:assistant$/),
      ]),
    );
    const parentMessages = await methods.getMessages({
      user: userId,
      conversationId: parentConversationId,
    });
    expect(parentMessages.filter((message) => message.isCreatedByUser)).toHaveLength(2);
    expect(parentMessages.filter((message) => !message.isCreatedByUser)).toHaveLength(3);

    await requesterStore.destroyTaskControlTransport();
  });

  it('serves a running child progress tree to a poll on another replica over Redis', async () => {
    const userId = new mongoose.Types.ObjectId().toString();
    const tenantId = 'tenant-progress';
    const parentConversationId = randomUUID();
    await saveParent(userId, tenantId, parentConversationId);
    const ownerStore = new SubagentThreadTaskStore(methods);
    const requesterStore = new SubagentThreadTaskStore(methods);
    taskStores.push(ownerStore, requesterStore);
    const namespace = `subagent-progress-${randomUUID()}`;
    await ownerStore.configureTaskControlTransport(
      await createRoutingTransport('progress-owner', namespace),
    );
    await requesterStore.configureTaskControlTransport(
      await createRoutingTransport('progress-requester', namespace),
    );
    const config = buildSubagentThreadTaskConfig(ownerStore, {
      userId,
      tenantId,
      parentConversationId,
    });
    let finish = (_value: { content: string }): void => undefined;
    let reported = false;
    const started = accepted(
      config.store.start(
        taskRequest(config.scopeId, 'review the change', 'reviewer', async (runtime) => {
          const base = {
            runId: 'root-run',
            subagentRunId: runtime.taskId,
            subagentType: 'reviewer',
            subagentAgentId: 'agent-reviewer',
            timestamp: new Date().toISOString(),
          };
          runtime.reportProgress({
            ...base,
            phase: 'run_step',
            data: {
              id: 'step-1',
              stepDetails: {
                type: 'tool_calls',
                tool_calls: [
                  {
                    id: 'call-1',
                    name: 'bash_tool',
                    args: { intent: 'Running the suite', env: 'KEY=1' },
                  },
                ],
              },
            },
          });
          reported = true;
          return new Promise<{ content: string }>((resolve) => (finish = resolve));
        }),
      ),
    );
    await waitUntil(() => (reported ? true : undefined));

    const routed = await waitUntil(async () => {
      const claim = await requesterStore.claimTask(config.scopeId, started.task.taskId);
      return claim.status === 'running' && snapshotActivity(claim.task) != null ? claim : undefined;
    });
    const tree = routed.status === 'running' ? snapshotActivity(routed.task) : undefined;
    expect(tree?.root.turns[0].children[0]).toMatchObject({
      kind: 'tool',
      name: 'bash_tool',
      label: 'Running the suite',
      status: 'running',
    });
    expect(JSON.stringify(tree)).not.toContain('KEY=1');
    const [listed] = await requesterStore.listTasks(config.scopeId);
    expect(snapshotActivitySummary(listed)).toMatchObject({ turns: 1, tools: 1 });

    const otherTenant = buildSubagentThreadTaskConfig(requesterStore, {
      userId,
      tenantId: 'tenant-other',
      parentConversationId,
    });
    await expect(
      requesterStore.claimTask(otherTenant.scopeId, started.task.taskId),
    ).resolves.toEqual({ status: 'not_found' });

    finish({ content: 'Review complete.' });
    await requesterStore.destroyTaskControlTransport();
  });
});