Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): collapse redelivered rows in the activity feed and agent list (#4695)
Gergő Móricz3cef4ced197d
diff --git a/apps/api/src/controllers/v1/__tests__/activity.test.ts b/apps/api/src/controllers/v1/__tests__/activity.test.tsindex d0b9b8ac4..62ce61eae 100644--- a/apps/api/src/controllers/v1/__tests__/activity.test.ts+++ b/apps/api/src/controllers/v1/__tests__/activity.test.ts@@ -104,6 +104,7 @@ describe("activityController", () => {expect(mocks.query).toHaveBeenCalledOnce();const options = mocks.query.mock.calls[0][0];expect(options.query).toContain("FROM requests");+ expect(options.query).toContain("LIMIT 1 BY id");expect(options.query).toContain("team_id = {teamId: UUID}");expect(options.query).toContain("created_at >= {windowStart: DateTime64(3)}",diff --git a/apps/api/src/controllers/v1/activity.ts b/apps/api/src/controllers/v1/activity.tsindex 82d763c51..860739734 100644--- a/apps/api/src/controllers/v1/activity.ts+++ b/apps/api/src/controllers/v1/activity.ts@@ -147,6 +147,7 @@ export async function activityController(FROM requestsWHERE ${conditions.join(" AND ")}ORDER BY created_at DESC, id DESC+ LIMIT 1 BY idLIMIT {limit: UInt32}`,query_params: queryParams,diff --git a/apps/api/src/controllers/v2/agent-list.ts b/apps/api/src/controllers/v2/agent-list.tsindex 7bf3206e2..e3abaf940 100644--- a/apps/api/src/controllers/v2/agent-list.ts+++ b/apps/api/src/controllers/v2/agent-list.ts@@ -44,6 +44,22 @@ export async function agentListController(});}+ // Tiebreaker for agents created in the same millisecond as the cursor. A+ // `next` link always carries it; a hand-built `before` may omit it.+ const beforeId = req.query.beforeId ? String(req.query.beforeId) : undefined;+ if (+ beforeId !== undefined &&+ (parsedBefore === undefined ||+ !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(+ beforeId,+ ))+ ) {+ return res.status(400).json({+ success: false,+ error: "Invalid beforeId cursor.",+ });+ }+if (!config.USE_DB_AUTHENTICATION) {return res.status(501).json({success: false,@@ -91,18 +107,28 @@ export async function agentListController();})(),(async () => {+ // Keyset pagination on (created_at, id), the same order the rows are+ // returned in, so agents created in the same millisecond as the last+ // one on a page are not skipped. Millisecond precision matches the+ // `before` value the `next` link carries; the old whole-second cursor+ // dropped every other agent created in that second.+ const cursorPredicate =+ beforeId !== undefined+ ? "(created_at, id) < ({before: DateTime64(3)}, {beforeId: UUID})"+ : "created_at < {before: DateTime64(3)}";const requestsRes = await clickhouseClient.query({- query:- "SELECT id, created_at, target_hint, origin, integration FROM requests WHERE team_id = {teamId: UUID} AND kind = 'agent' AND created_at < {before: DateTime} ORDER BY created_at DESC LIMIT {limit: UInt32};",+ query: `SELECT id, created_at, target_hint, origin, integration FROM requests WHERE team_id = {teamId: UUID} AND kind = 'agent' AND ${cursorPredicate} ORDER BY created_at DESC, id DESC LIMIT 1 BY id LIMIT {limit: UInt32};`,query_params: {teamId: req.auth.team_id,// Fetch one extra row so we can tell whether another page exists// instead of emitting a next cursor whenever the page is full.limit: limit + 1,- before: (parsedBefore !== undefined- ? new Date(parsedBefore).toISOString()- : new Date().toISOString()- ).split(".")[0], // slice .nnnZ off the end of the ISO string because clickhouse hates it+ // "YYYY-MM-DD hh:mm:ss.mmm": what a DateTime64(3) parameter takes.+ before: new Date(parsedBefore ?? Date.now())+ .toISOString()+ .replace("T", " ")+ .replace("Z", ""),+ ...(beforeId !== undefined ? { beforeId } : {}),},format: "JSONEachRow",});@@ -130,10 +156,11 @@ export async function agentListController();// `agents` is keyed by (team_id, id); the team filter keeps this a- // primary-key read instead of a scan.+ // primary-key read instead of a scan. The table keeps the latest+ // publication per id on merge; argMax picks the same row before it.const agentsRes = await clickhouseClient.query({query:- "SELECT id, options, is_successful, error FROM agents WHERE team_id = {teamId: UUID} AND id IN {ids: Array(UUID)};",+ "SELECT id, argMax(options, _publish_time) AS options, argMax(is_successful, _publish_time) AS is_successful, argMax(error, _publish_time) AS error FROM agents WHERE team_id = {teamId: UUID} AND id IN {ids: Array(UUID)} GROUP BY id;",query_params: {teamId: req.auth.team_id,ids: bareRequests.map(x => x.id),@@ -278,7 +305,7 @@ export async function agentListController(success: true,agents: page,next: hasMore- ? `${req.protocol}://${req.host}/v2/agent?before=${new Date(page.slice(-1)[0].createdAt).valueOf()}`+ ? `${req.protocol}://${req.host}/v2/agent?before=${new Date(page.slice(-1)[0].createdAt).valueOf()}&beforeId=${encodeURIComponent(page.slice(-1)[0].id)}`: undefined,});}