Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

12.5 KB
Raw Blame
import { and, eq, inArray } from "drizzle-orm";
import { config } from "../../config";
import { db } from "../../db/connection";
import { agent_session_settings } from "../../db/schema";
import { clickhouseClient } from "../../lib/clickhouse-client";
import { AgentListResponse, ErrorResponse, RequestWithAuth } from "./types";
import { Response } from "express";

type RecentAgent = {
  id: string;
  options: any;
  createdAt: string;
  status:
    | "processing"
    | "succeeded"
    | "failed"
    | "cancelled"
    | "refused"
    | "credit_limit_reached";
};

// ClickHouse returns the `agents.options` column as a JSON string, not an
// object. Parse it so the option fields can be read. A value that does not
// parse to an object reads as no options.
function parseAgentOptions(options: unknown): Record<string, any> {
  let parsed = options;
  if (typeof parsed === "string") {
    try {
      parsed = JSON.parse(parsed);
    } catch {
      return {};
    }
  }
  return parsed !== null && typeof parsed === "object" ? parsed : {};
}

// JSONEachRow can quote 64-bit integers, so a turn can arrive as "2". A turn
// that a number cannot hold exactly is dropped, not rounded.
function threadTurnOf(value: unknown): number | undefined {
  const turn =
    typeof value === "number"
      ? value
      : typeof value === "string" && value.trim() !== ""
        ? Number(value)
        : NaN;
  return Number.isSafeInteger(turn) ? turn : undefined;
}

const ZDR_TARGET_HINT = "<redacted due to zero data retention>";

function isZdrOptions(options: any): boolean {
  return options?.zeroDataRetention === true;
}

// A zero data retention run is listed with its model, effort and the flag
// only, whichever source the row comes from.
function withoutZdrContent(options: any): any {
  return isZdrOptions(options)
    ? {
        model: options.model,
        modelPreset: options.modelPreset,
        effort: options.effort,
        safe: options.safe,
        zeroDataRetention: true,
      }
    : options;
}

function safeOption(options: any): { safe?: true } {
  return options?.safe === true ? { safe: true } : {};
}

function zdrOptions(options: any): { zeroDataRetention?: true } {
  return isZdrOptions(options) ? { zeroDataRetention: true } : {};
}

// Each turn of a thread is its own run, so clients need the thread to list
// a conversation once.
function threadOptions(options: any): {
  threadId?: string;
  threadTurn?: number;
} {
  const threadTurn = threadTurnOf(options?.threadTurn);
  return {
    ...(typeof options?.threadId === "string" && {
      threadId: options.threadId,
    }),
    ...(threadTurn !== undefined && { threadTurn }),
  };
}

export async function agentListController(
  req: RequestWithAuth<{}, AgentListResponse>,
  res: Response<AgentListResponse>,
) {
  const limit = 20;

  // Number() (not parseInt) so malformed values like "123abc" are rejected
  // instead of silently truncated.
  const parsedBefore = req.query.before
    ? Number(req.query.before as string)
    : undefined;

  if (
    parsedBefore !== undefined &&
    (isNaN(parsedBefore) ||
      !isFinite(parsedBefore) ||
      !Number.isInteger(parsedBefore) ||
      parsedBefore < 0)
  ) {
    return res.status(400).json({
      success: false,
      error: "Invalid before timestamp.",
    });
  }

  // Tiebreaker for agents created in the same millisecond as the cursor. A
  // `next` link always carries it; a hand-built `before` may omit it.
  const beforeId = req.query.beforeId ? String(req.query.beforeId) : undefined;
  if (
    beforeId !== undefined &&
    (parsedBefore === undefined ||
      !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
        beforeId,
      ))
  ) {
    return res.status(400).json({
      success: false,
      error: "Invalid beforeId cursor.",
    });
  }

  if (!config.USE_DB_AUTHENTICATION) {
    return res.status(501).json({
      success: false,
      error:
        "This endpoint is only available if your Firecrawl deployment is backed by a database.",
    });
  }

  if (!config.EXTRACT_V3_BETA_URL) {
    throw new Error("Agent beta is not enabled.");
  }

  if (clickhouseClient === null) {
    return res.status(501).json({
      success: false,
      error: "This endpoint is only available if ClickHouse is configured.",
    });
  }

  const [recentResult, dbResult] = await Promise.allSettled([
    (async () => {
      const recentRes = await fetch(
        config.EXTRACT_V3_BETA_URL +
          "/internal/recent-agents-for-team?teamId=" +
          req.auth.team_id,
        {
          method: "GET",
          headers: {
            Authorization: `Bearer ${config.AGENT_INTEROP_SECRET}`,
          },
        },
      );

      if (!recentRes.ok) {
        throw new Error(
          "Unexpected response: " +
            recentRes.statusText +
            " " +
            (await recentRes.text()),
        );
      }

      return new Map(
        (await recentRes.json()).map((x: RecentAgent) => [x.id, x]),
      );
    })(),
    (async () => {
      // Keyset pagination on (created_at, id), the same order the rows are
      // returned in, so agents created in the same millisecond as the last
      // one on a page are not skipped. Millisecond precision matches the
      // `before` value the `next` link carries; the old whole-second cursor
      // dropped every other agent created in that second.
      const cursorPredicate =
        beforeId !== undefined
          ? "(created_at, id) < ({before: DateTime64(3)}, {beforeId: UUID})"
          : "created_at < {before: DateTime64(3)}";
      const requestsRes = await clickhouseClient.query({
        query: `SELECT id, created_at, target_hint, origin, integration FROM requests WHERE team_id = {teamId: UUID} AND kind = 'agent' AND ${cursorPredicate} ORDER BY created_at DESC, id DESC LIMIT 1 BY id LIMIT {limit: UInt32};`,
        query_params: {
          teamId: req.auth.team_id,
          // Fetch one extra row so we can tell whether another page exists
          // instead of emitting a next cursor whenever the page is full.
          limit: limit + 1,
          // "YYYY-MM-DD hh:mm:ss.mmm": what a DateTime64(3) parameter takes.
          before: new Date(parsedBefore ?? Date.now())
            .toISOString()
            .replace("T", " ")
            .replace("Z", ""),
          ...(beforeId !== undefined ? { beforeId } : {}),
        },
        format: "JSONEachRow",
      });

      const bareRequests: {
        id: string;
        createdAt: Date;
        targetHint: string;
        origin: string;
        integration: string | null | undefined;
      }[] = (await requestsRes.json()).map(
        (x: {
          id: string;
          created_at: string;
          target_hint: string;
          origin: string;
          integration: string;
        }) => ({
          id: x.id,
          createdAt: new Date(x.created_at),
          targetHint: x.target_hint,
          origin: x.origin,
          integration: x.integration,
        }),
      );

      // `agents` is keyed by (team_id, id); the team filter keeps this a
      // primary-key read instead of a scan. The table keeps the latest
      // publication per id on merge; argMax picks the same row before it.
      const agentsRes = await clickhouseClient.query({
        query:
          "SELECT id, argMax(options, _publish_time) AS options, argMax(is_successful, _publish_time) AS is_successful, argMax(error, _publish_time) AS error FROM agents WHERE team_id = {teamId: UUID} AND id IN {ids: Array(UUID)} GROUP BY id;",
        query_params: {
          teamId: req.auth.team_id,
          ids: bareRequests.map(x => x.id),
        },
        format: "JSONEachRow",
      });
      const agentResults: {
        id: string;
        options: any;
        isSuccessful: boolean;
        error: string | null;
      }[] = (await agentsRes.json()).map(
        (x: {
          id: string;
          options: any;
          is_successful: boolean;
          error: string | null;
        }) => ({
          id: x.id,
          options: parseAgentOptions(x.options),
          isSuccessful: x.is_successful,
          error: x.error,
        }),
      );
      const agentMap = new Map(agentResults.map(x => [x.id, x]));

      return new Map(
        bareRequests
          .map(x => ({ ...x, agent: agentMap.get(x.id) }))
          .map(x => [x.id, x]),
      );
    })(),
  ]);

  const recentAgents: Map<string, RecentAgent> =
    recentResult.status === "fulfilled" ? recentResult.value : new Map();

  // Recent agents are merged into every response, so without this filter they
  // would repeat on every paginated page. Only include the ones that fall
  // within the requested page range.
  if (parsedBefore !== undefined) {
    for (const [id, agent] of recentAgents) {
      if (new Date(agent.createdAt).valueOf() >= parsedBefore) {
        recentAgents.delete(id);
      }
    }
  }
  if (dbResult.status === "rejected") {
    throw dbResult.reason;
  }
  const dbAgents = dbResult.value;

  const allIds = [...new Set([...recentAgents.keys(), ...dbAgents.keys()])];

  const sessionSettings = await db
    .select()
    .from(agent_session_settings)
    .where(
      and(
        inArray(agent_session_settings.session_id, allIds),
        eq(agent_session_settings.team_id, req.auth.team_id),
      ),
    );

  const sessionSettingsMap = new Map(
    sessionSettings.map(x => [x.session_id, x]),
  );

  const agents: Exclude<AgentListResponse, ErrorResponse>["agents"] = [];

  for (const id of allIds) {
    const recent = recentAgents.get(id);
    const db = dbAgents.get(id);
    const settings = sessionSettingsMap.get(id);
    const recentOptions = withoutZdrContent(recent?.options);
    const dbOptions = db?.agent ? withoutZdrContent(db.agent.options) : null;

    agents.push({
      id: db?.id ?? recent!.id,
      createdAt: db?.createdAt.toISOString() ?? recent!.createdAt,
      targetHint:
        db?.targetHint ??
        (isZdrOptions(recentOptions)
          ? ZDR_TARGET_HINT
          : recentOptions
            ? (recentOptions.urls?.[0] ?? recentOptions.prompt ?? "")
            : ""),
      origin: db?.origin ?? "api", // recent cannot know origin so we lie
      integration: db?.integration ?? undefined, // recent cannot know integration so we omit
      settings: {
        hidden: settings?.hidden ?? false,
        starred: settings?.starred ?? false,
        label: settings?.label ?? undefined,
      },
      status: db
        ? db.agent
          ? db.agent.isSuccessful
            ? "completed"
            : "failed"
          : "processing"
        : recent!.status === "processing"
          ? "processing"
          : recent!.status === "succeeded"
            ? "completed"
            : "failed",
      options: dbOptions
        ? {
            urls: dbOptions.urls ?? undefined,
            prompt: dbOptions.prompt ?? "",
            schema: dbOptions.schema ?? undefined,
            model: dbOptions.model ?? "spark-1-pro",
            effort:
              dbOptions.effort ??
              (dbOptions.model === "spark-2" ? "medium" : undefined),
            ...safeOption(dbOptions),
            ...threadOptions(dbOptions),
            ...zdrOptions(dbOptions),
          }
        : recentOptions
          ? {
              urls: recentOptions.urls ?? undefined,
              prompt: recentOptions.prompt ?? "",
              schema: recentOptions.schema ?? undefined,
              model: recentOptions.modelPreset ?? "spark-1-pro",
              effort:
                recentOptions.effort ??
                (recentOptions.modelPreset === "spark-2"
                  ? "medium"
                  : undefined),
              ...safeOption(recentOptions),
              ...threadOptions(recentOptions),
              ...zdrOptions(recentOptions),
            }
          : undefined,
    });
  }

  // The merged list is recent agents followed by ClickHouse rows; sort so the
  // page is globally ordered by creation time and the next cursor (taken from
  // the last entry) is correct.
  agents.sort(
    (a, b) => new Date(b.createdAt).valueOf() - new Date(a.createdAt).valueOf(),
  );

  // The ClickHouse query fetched limit + 1 rows and recent agents are merged
  // on top, so anything past limit means another page exists. Dropped entries
  // are older than the new cursor and resurface on the next page.
  const hasMore = agents.length > limit;
  const page = agents.slice(0, limit);

  return res.json({
    success: true,
    agents: page,
    next: hasMore
      ? `${req.protocol}://${req.host}/v2/agent?before=${new Date(page.slice(-1)[0].createdAt).valueOf()}&beforeId=${encodeURIComponent(page.slice(-1)[0].id)}`
      : undefined,
  });
}