Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): collapse redelivered rows in the activity feed and agent list (#4695)

Gergő Móricz3cef4ced197d
diff --git a/apps/api/src/controllers/v1/__tests__/activity.test.ts b/apps/api/src/controllers/v1/__tests__/activity.test.ts
index d0b9b8ac4..62ce61eae 100644
--- a/apps/api/src/controllers/v1/__tests__/activity.test.ts
+++ b/apps/api/src/controllers/v1/__tests__/activity.test.ts
@@ -104,6 +104,7 @@ describe("activityController", () => {
expect(mocks.query).toHaveBeenCalledOnce();
const options = mocks.query.mock.calls[0][0];
expect(options.query).toContain("FROM requests");
+ expect(options.query).toContain("LIMIT 1 BY id");
expect(options.query).toContain("team_id = {teamId: UUID}");
expect(options.query).toContain(
"created_at >= {windowStart: DateTime64(3)}",
diff --git a/apps/api/src/controllers/v1/activity.ts b/apps/api/src/controllers/v1/activity.ts
index 82d763c51..860739734 100644
--- a/apps/api/src/controllers/v1/activity.ts
+++ b/apps/api/src/controllers/v1/activity.ts
@@ -147,6 +147,7 @@ export async function activityController(
FROM requests
WHERE ${conditions.join(" AND ")}
ORDER BY created_at DESC, id DESC
+ LIMIT 1 BY id
LIMIT {limit: UInt32}
`,
query_params: queryParams,
diff --git a/apps/api/src/controllers/v2/agent-list.ts b/apps/api/src/controllers/v2/agent-list.ts
index 7bf3206e2..e3abaf940 100644
--- a/apps/api/src/controllers/v2/agent-list.ts
+++ b/apps/api/src/controllers/v2/agent-list.ts
@@ -44,6 +44,22 @@ export async function agentListController(
});
}
+ // Tiebreaker for agents created in the same millisecond as the cursor. A
+ // `next` link always carries it; a hand-built `before` may omit it.
+ const beforeId = req.query.beforeId ? String(req.query.beforeId) : undefined;
+ if (
+ beforeId !== undefined &&
+ (parsedBefore === undefined ||
+ !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
+ beforeId,
+ ))
+ ) {
+ return res.status(400).json({
+ success: false,
+ error: "Invalid beforeId cursor.",
+ });
+ }
+
if (!config.USE_DB_AUTHENTICATION) {
return res.status(501).json({
success: false,
@@ -91,18 +107,28 @@ export async function agentListController(
);
})(),
(async () => {
+ // Keyset pagination on (created_at, id), the same order the rows are
+ // returned in, so agents created in the same millisecond as the last
+ // one on a page are not skipped. Millisecond precision matches the
+ // `before` value the `next` link carries; the old whole-second cursor
+ // dropped every other agent created in that second.
+ const cursorPredicate =
+ beforeId !== undefined
+ ? "(created_at, id) < ({before: DateTime64(3)}, {beforeId: UUID})"
+ : "created_at < {before: DateTime64(3)}";
const requestsRes = await clickhouseClient.query({
- query:
- "SELECT id, created_at, target_hint, origin, integration FROM requests WHERE team_id = {teamId: UUID} AND kind = 'agent' AND created_at < {before: DateTime} ORDER BY created_at DESC LIMIT {limit: UInt32};",
+ query: `SELECT id, created_at, target_hint, origin, integration FROM requests WHERE team_id = {teamId: UUID} AND kind = 'agent' AND ${cursorPredicate} ORDER BY created_at DESC, id DESC LIMIT 1 BY id LIMIT {limit: UInt32};`,
query_params: {
teamId: req.auth.team_id,
// Fetch one extra row so we can tell whether another page exists
// instead of emitting a next cursor whenever the page is full.
limit: limit + 1,
- before: (parsedBefore !== undefined
- ? new Date(parsedBefore).toISOString()
- : new Date().toISOString()
- ).split(".")[0], // slice .nnnZ off the end of the ISO string because clickhouse hates it
+ // "YYYY-MM-DD hh:mm:ss.mmm": what a DateTime64(3) parameter takes.
+ before: new Date(parsedBefore ?? Date.now())
+ .toISOString()
+ .replace("T", " ")
+ .replace("Z", ""),
+ ...(beforeId !== undefined ? { beforeId } : {}),
},
format: "JSONEachRow",
});
@@ -130,10 +156,11 @@ export async function agentListController(
);
// `agents` is keyed by (team_id, id); the team filter keeps this a
- // primary-key read instead of a scan.
+ // primary-key read instead of a scan. The table keeps the latest
+ // publication per id on merge; argMax picks the same row before it.
const agentsRes = await clickhouseClient.query({
query:
- "SELECT id, options, is_successful, error FROM agents WHERE team_id = {teamId: UUID} AND id IN {ids: Array(UUID)};",
+ "SELECT id, argMax(options, _publish_time) AS options, argMax(is_successful, _publish_time) AS is_successful, argMax(error, _publish_time) AS error FROM agents WHERE team_id = {teamId: UUID} AND id IN {ids: Array(UUID)} GROUP BY id;",
query_params: {
teamId: req.auth.team_id,
ids: bareRequests.map(x => x.id),
@@ -278,7 +305,7 @@ export async function agentListController(
success: true,
agents: page,
next: hasMore
- ? `${req.protocol}://${req.host}/v2/agent?before=${new Date(page.slice(-1)[0].createdAt).valueOf()}`
+ ? `${req.protocol}://${req.host}/v2/agent?before=${new Date(page.slice(-1)[0].createdAt).valueOf()}&beforeId=${encodeURIComponent(page.slice(-1)[0].id)}`
: undefined,
});
}