Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(shared): chunk queries to stay under D1's 100-bound-param limit

Gener21b40b4cfff9
diff --git a/src/shared/src/db/queries/_chunk.ts b/src/shared/src/db/queries/_chunk.ts
new file mode 100644
index 000000000..32feba999
--- /dev/null
+++ b/src/shared/src/db/queries/_chunk.ts
@@ -0,0 +1,39 @@
+// Cloudflare D1 rejects any single SQL statement binding more than 100
+// parameters (SQLite SQLITE_MAX_VARIABLE_NUMBER = 100 in the D1 build). Queries
+// that build a WHERE `inArray(col, ids)` or a bulk INSERT `values(rows)` from an
+// unbounded runtime array must slice the array so no single statement crosses
+// the limit. These helpers centralize that arithmetic — spec, not incidental.
+export const D1_MAX_BIND_PARAMS = 100;
+
+// Safe slice size for an `inArray(col, ids)` whose statement also binds a
+// handful of fixed WHERE params (userId, read flag, join-baseline guard,
+// correlated subselects, ...). None of the chunked sites carry more than ~10
+// such fixed params, so 90 keeps every statement's total <= 100. Chunk id
+// arrays at this size, not at D1_MAX_BIND_PARAMS, so the fixed params fit.
+export const D1_MAX_IN_PARAMS = 90;
+
+/**
+ * Split `arr` into ordered slices of at most `size` elements. `chunk([], n)`
+ * returns `[]`; order is preserved. `size` must be >= 1.
+ */
+export function chunk<T>(arr: T[], size: number): T[][] {
+ if (size < 1) throw new Error("chunk size must be >= 1");
+ const out: T[][] = [];
+ for (let i = 0; i < arr.length; i += size) {
+ out.push(arr.slice(i, i + size));
+ }
+ return out;
+}
+
+/**
+ * Max rows per bulk-insert statement given the REAL number of bind params each
+ * row emits. `paramsPerRow` is the emitted bind count — Drizzle binds every
+ * non-disabled column of each row, including a `$defaultFn` primary key and
+ * columns with a literal `default(...)`, so count from the built statement, not
+ * the "logical" columns. floor(100 / paramsPerRow) keeps rows*paramsPerRow
+ * <= D1_MAX_BIND_PARAMS (the limit rejects > 100, so hitting exactly 100 is fine).
+ */
+export function maxRowsPerInsert(paramsPerRow: number): number {
+ if (paramsPerRow < 1) throw new Error("paramsPerRow must be >= 1");
+ return Math.floor(D1_MAX_BIND_PARAMS / paramsPerRow);
+}
diff --git a/src/shared/src/db/queries/community/agent-inbox.ts b/src/shared/src/db/queries/community/agent-inbox.ts
index c2c131d10..5525e8d5c 100644
--- a/src/shared/src/db/queries/community/agent-inbox.ts
+++ b/src/shared/src/db/queries/community/agent-inbox.ts
@@ -37,6 +37,7 @@ import { listVisibleChannelIdsForUser } from "./channel";
import { listParticipatingThreadIds } from "./thread";
import { getMessagesByIdsInScope, type MessageScope } from "./message";
import { isThread, isForumPost } from "../../../utils/community-roles";
+import { chunk, D1_MAX_IN_PARAMS } from "../_chunk";
type RawAgentMessage = {
id: string;
@@ -421,10 +422,17 @@ async function listAgentAllowedChannelIds(db: Database, botUserId: string): Prom
if (visibleChannelIds.length === 0) return dmChannelIds;
- const typeRows = await db
- .select({ id: communityChannel.id, type: communityChannel.type })
- .from(communityChannel)
- .where(inArray(communityChannel.id, visibleChannelIds));
+ // Chunk the `inArray` for D1's 100-param limit; no order/limit → concat.
+ const typeRows = (
+ await Promise.all(
+ chunk(visibleChannelIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select({ id: communityChannel.id, type: communityChannel.type })
+ .from(communityChannel)
+ .where(inArray(communityChannel.id, ids))
+ )
+ )
+ ).flat();
const narrowIds = typeRows
.filter((r) => isThread(r.type) || isForumPost(r.type))
.map((r) => r.id);
@@ -477,50 +485,68 @@ export async function listUnreadMessagesForAgent(
opts: { max: number }
): Promise<RawAgentMessage[]> {
const allowedChannelIds = await listAgentAllowedChannelIds(db, botUserId);
-
- const rows = await db
- .select({
- ...AGENT_MESSAGE_COLUMNS,
- lastReadSeq: sql<number>`COALESCE(${communityReadState.lastReadSeq}, 0)`,
- })
- .from(communityMessage)
- .leftJoin(
- communityReadState,
- and(
- eq(communityReadState.userId, botUserId),
- eq(communityReadState.channelId, communityMessage.channelId)
+ if (allowedChannelIds.length === 0) return [];
+
+ // D1 caps a statement at 100 bound params, and `allowedChannelIds` is
+ // unbounded (a bot allowed into >100 channels). Chunk the `inArray` and merge:
+ // each chunk runs the SAME order+limit; since chunks partition channel ids,
+ // each chunk's result is a subsequence of the global (channelId, seq) order,
+ // so the global top-`max` is contained in the union of per-chunk top-`max`.
+ // Re-sort the union and slice to `max`. Both sort keys are in the projection.
+ const runChunk = (ids: string[]) =>
+ db
+ .select({
+ ...AGENT_MESSAGE_COLUMNS,
+ lastReadSeq: sql<number>`COALESCE(${communityReadState.lastReadSeq}, 0)`,
+ })
+ .from(communityMessage)
+ .leftJoin(
+ communityReadState,
+ and(
+ eq(communityReadState.userId, botUserId),
+ eq(communityReadState.channelId, communityMessage.channelId)
+ )
)
- )
- .leftJoin(communityChannel, eq(communityChannel.id, communityMessage.channelId))
- .leftJoin(
- communityChannelMember,
- and(
- eq(communityChannelMember.channelId, communityMessage.channelId),
- eq(communityChannelMember.userId, botUserId),
- eq(communityChannelMember.relation, "access")
+ .leftJoin(communityChannel, eq(communityChannel.id, communityMessage.channelId))
+ .leftJoin(
+ communityChannelMember,
+ and(
+ eq(communityChannelMember.channelId, communityMessage.channelId),
+ eq(communityChannelMember.userId, botUserId),
+ eq(communityChannelMember.relation, "access")
+ )
)
- )
- .leftJoin(
- communityServerMember,
- and(
- eq(communityServerMember.serverId, communityChannel.serverId),
- eq(communityServerMember.userId, botUserId)
+ .leftJoin(
+ communityServerMember,
+ and(
+ eq(communityServerMember.serverId, communityChannel.serverId),
+ eq(communityServerMember.userId, botUserId)
+ )
)
- )
- .where(
- and(
- ne(communityMessage.authorId, botUserId),
- sql`${communityMessage.seq} > COALESCE(${communityReadState.lastReadSeq}, 0)`,
- allowedChannelIds.length > 0
- ? inArray(communityMessage.channelId, allowedChannelIds)
- : sql`1 = 0`,
- channelJoinBaselineGuard
+ .where(
+ and(
+ ne(communityMessage.authorId, botUserId),
+ sql`${communityMessage.seq} > COALESCE(${communityReadState.lastReadSeq}, 0)`,
+ inArray(communityMessage.channelId, ids),
+ channelJoinBaselineGuard
+ )
)
- )
- .orderBy(asc(communityMessage.channelId), asc(communityMessage.seq))
- .limit(opts.max);
-
- return rows.map(({ lastReadSeq: _lastReadSeq, ...rest }) => rest);
+ .orderBy(asc(communityMessage.channelId), asc(communityMessage.seq))
+ .limit(opts.max);
+
+ const merged = (
+ await Promise.all(chunk(allowedChannelIds, D1_MAX_IN_PARAMS).map(runChunk))
+ ).flat();
+ merged.sort((a, b) =>
+ a.channelId === b.channelId
+ ? a.seq - b.seq
+ : a.channelId < b.channelId
+ ? -1
+ : 1
+ );
+ return merged
+ .slice(0, opts.max)
+ .map(({ lastReadSeq: _lastReadSeq, ...rest }) => rest);
}
/**
@@ -609,57 +635,65 @@ export type InboxSnapshotRow = {
*/
export async function getInboxSnapshotForAgent(db: Database, botUserId: string): Promise<InboxSnapshotRow[]> {
const allowedChannelIds = await listAgentAllowedChannelIds(db, botUserId);
+ if (allowedChannelIds.length === 0) return [];
- const rows = await db
- .select({
- channelId: communityMessage.channelId,
- pendingCount: sql<number>`COUNT(*)`,
- firstPendingSeq: sql<number>`MIN(${communityMessage.seq})`,
- latestSeq: sql<number>`MAX(${communityMessage.seq})`,
- latestSenderId: sql<string>`(SELECT author_id FROM community_message m2
- WHERE m2.channel_id = ${communityMessage.channelId}
- ORDER BY m2.seq DESC LIMIT 1)`,
- mentionCount: sql<number>`(SELECT COUNT(*) FROM community_mention cm
- INNER JOIN community_message m3 ON m3.id = cm.message_id
- WHERE cm.user_id = ${botUserId} AND cm.kind = 'mention'
- AND m3.channel_id = ${communityMessage.channelId}
- AND m3.seq > COALESCE(${communityReadState.lastReadSeq}, 0))`,
- })
- .from(communityMessage)
- .leftJoin(
- communityReadState,
- and(
- eq(communityReadState.userId, botUserId),
- eq(communityReadState.channelId, communityMessage.channelId)
+ // Chunk the `inArray` for D1's 100-param limit. GROUP BY channelId partitions
+ // cleanly across chunks — a channel id lands in exactly one chunk, so its
+ // COUNT/MIN/MAX and the correlated subselects are complete within that chunk.
+ // Concat is loss-free (no channel appears in two chunks).
+ const runChunk = (ids: string[]) =>
+ db
+ .select({
+ channelId: communityMessage.channelId,
+ pendingCount: sql<number>`COUNT(*)`,
+ firstPendingSeq: sql<number>`MIN(${communityMessage.seq})`,
+ latestSeq: sql<number>`MAX(${communityMessage.seq})`,
+ latestSenderId: sql<string>`(SELECT author_id FROM community_message m2
+ WHERE m2.channel_id = ${communityMessage.channelId}
+ ORDER BY m2.seq DESC LIMIT 1)`,
+ mentionCount: sql<number>`(SELECT COUNT(*) FROM community_mention cm
+ INNER JOIN community_message m3 ON m3.id = cm.message_id
+ WHERE cm.user_id = ${botUserId} AND cm.kind = 'mention'
+ AND m3.channel_id = ${communityMessage.channelId}
+ AND m3.seq > COALESCE(${communityReadState.lastReadSeq}, 0))`,
+ })
+ .from(communityMessage)
+ .leftJoin(
+ communityReadState,
+ and(
+ eq(communityReadState.userId, botUserId),
+ eq(communityReadState.channelId, communityMessage.channelId)
+ )
)
- )
- .leftJoin(communityChannel, eq(communityChannel.id, communityMessage.channelId))
- .leftJoin(
- communityChannelMember,
- and(
- eq(communityChannelMember.channelId, communityMessage.channelId),
- eq(communityChannelMember.userId, botUserId),
- eq(communityChannelMember.relation, "access")
+ .leftJoin(communityChannel, eq(communityChannel.id, communityMessage.channelId))
+ .leftJoin(
+ communityChannelMember,
+ and(
+ eq(communityChannelMember.channelId, communityMessage.channelId),
+ eq(communityChannelMember.userId, botUserId),
+ eq(communityChannelMember.relation, "access")
+ )
)
- )
- .leftJoin(
- communityServerMember,
- and(
- eq(communityServerMember.serverId, communityChannel.serverId),
- eq(communityServerMember.userId, botUserId)
+ .leftJoin(
+ communityServerMember,
+ and(
+ eq(communityServerMember.serverId, communityChannel.serverId),
+ eq(communityServerMember.userId, botUserId)
+ )
)
- )
- .where(
- and(
- ne(communityMessage.authorId, botUserId),
- sql`${communityMessage.seq} > COALESCE(${communityReadState.lastReadSeq}, 0)`,
- allowedChannelIds.length > 0
- ? inArray(communityMessage.channelId, allowedChannelIds)
- : sql`1 = 0`,
- channelJoinBaselineGuard
+ .where(
+ and(
+ ne(communityMessage.authorId, botUserId),
+ sql`${communityMessage.seq} > COALESCE(${communityReadState.lastReadSeq}, 0)`,
+ inArray(communityMessage.channelId, ids),
+ channelJoinBaselineGuard
+ )
)
- )
- .groupBy(communityMessage.channelId);
+ .groupBy(communityMessage.channelId);
+
+ const rows = (
+ await Promise.all(chunk(allowedChannelIds, D1_MAX_IN_PARAMS).map(runChunk))
+ ).flat();
if (rows.length === 0) return [];
@@ -747,50 +781,60 @@ export async function getLatestUnreadMessageForAgent(
botUserId: string
): Promise<{ messageId: string } | null> {
const allowedChannelIds = await listAgentAllowedChannelIds(db, botUserId);
+ if (allowedChannelIds.length === 0) return null;
- const rows = await db
- .select({
- id: communityMessage.id,
- })
- .from(communityMessage)
- .leftJoin(
- communityReadState,
- and(
- eq(communityReadState.userId, botUserId),
- eq(communityReadState.channelId, communityMessage.channelId)
+ // Chunk the `inArray` for D1's 100-param limit; each chunk returns its own
+ // newest-by-createdAt winner (LIMIT 1). To pick the GLOBAL winner across
+ // chunks we must compare `createdAt`, so it's added to the projection (the
+ // original selected only `id` and couldn't be merged).
+ const runChunk = (ids: string[]) =>
+ db
+ .select({
+ id: communityMessage.id,
+ createdAt: communityMessage.createdAt,
+ })
+ .from(communityMessage)
+ .leftJoin(
+ communityReadState,
+ and(
+ eq(communityReadState.userId, botUserId),
+ eq(communityReadState.channelId, communityMessage.channelId)
+ )
)
- )
- .leftJoin(communityChannel, eq(communityChannel.id, communityMessage.channelId))
- .leftJoin(
- communityChannelMember,
- and(
- eq(communityChannelMember.channelId, communityMessage.channelId),
- eq(communityChannelMember.userId, botUserId),
- eq(communityChannelMember.relation, "access")
+ .leftJoin(communityChannel, eq(communityChannel.id, communityMessage.channelId))
+ .leftJoin(
+ communityChannelMember,
+ and(
+ eq(communityChannelMember.channelId, communityMessage.channelId),
+ eq(communityChannelMember.userId, botUserId),
+ eq(communityChannelMember.relation, "access")
+ )
)
- )
- .leftJoin(
- communityServerMember,
- and(
- eq(communityServerMember.serverId, communityChannel.serverId),
- eq(communityServerMember.userId, botUserId)
+ .leftJoin(
+ communityServerMember,
+ and(
+ eq(communityServerMember.serverId, communityChannel.serverId),
+ eq(communityServerMember.userId, botUserId)
+ )
)
- )
- .where(
- and(
- ne(communityMessage.authorId, botUserId),
- sql`${communityMessage.seq} > COALESCE(${communityReadState.lastReadSeq}, 0)`,
- allowedChannelIds.length > 0
- ? inArray(communityMessage.channelId, allowedChannelIds)
- : sql`1 = 0`,
- channelJoinBaselineGuard
+ .where(
+ and(
+ ne(communityMessage.authorId, botUserId),
+ sql`${communityMessage.seq} > COALESCE(${communityReadState.lastReadSeq}, 0)`,
+ inArray(communityMessage.channelId, ids),
+ channelJoinBaselineGuard
+ )
)
- )
- .orderBy(desc(communityMessage.createdAt))
- .limit(1);
+ .orderBy(desc(communityMessage.createdAt))
+ .limit(1);
- const r = rows[0];
- return r ? { messageId: r.id } : null;
+ const winners = (
+ await Promise.all(chunk(allowedChannelIds, D1_MAX_IN_PARAMS).map(runChunk))
+ ).flat();
+ if (winners.length === 0) return null;
+ let best = winners[0]!;
+ for (const w of winners) if (w.createdAt > best.createdAt) best = w;
+ return { messageId: best.id };
}
/**
diff --git a/src/shared/src/db/queries/community/attachment.ts b/src/shared/src/db/queries/community/attachment.ts
index fc938b2ad..bd578a3aa 100644
--- a/src/shared/src/db/queries/community/attachment.ts
+++ b/src/shared/src/db/queries/community/attachment.ts
@@ -2,6 +2,7 @@ import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm";
import { nanoid } from "nanoid";
import { communityAttachment } from "../../community-schema";
import type { Database } from "../../index";
+import { chunk, D1_MAX_IN_PARAMS } from "../_chunk";
/**
* Insert an attachment row already tied to a message (human-composer path).
@@ -193,9 +194,29 @@ export async function listByMessageIds(
messageIds: string[]
) {
if (messageIds.length === 0) return [];
- return db
- .select()
- .from(communityAttachment)
- .where(inArray(communityAttachment.messageId, messageIds))
- .orderBy(asc(communityAttachment.position), asc(communityAttachment.createdAt));
+ // Live risk: inboxPull passes a page of up to 200 message ids (> 100). Chunk
+ // the `inArray` for D1's 100-param limit, concat, then re-sort globally by
+ // (position, createdAt) — both keys are in the projection, so per-chunk order
+ // alone would be wrong across chunks.
+ const rows = (
+ await Promise.all(
+ chunk(messageIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select()
+ .from(communityAttachment)
+ .where(inArray(communityAttachment.messageId, ids))
+ .orderBy(asc(communityAttachment.position), asc(communityAttachment.createdAt))
+ )
+ )
+ ).flat();
+ rows.sort((a, b) =>
+ a.position !== b.position
+ ? (a.position ?? 0) - (b.position ?? 0)
+ : a.createdAt < b.createdAt
+ ? -1
+ : a.createdAt > b.createdAt
+ ? 1
+ : 0
+ );
+ return rows;
}
diff --git a/src/shared/src/db/queries/community/bot.ts b/src/shared/src/db/queries/community/bot.ts
index c60d4ba7a..d33529c34 100644
--- a/src/shared/src/db/queries/community/bot.ts
+++ b/src/shared/src/db/queries/community/bot.ts
@@ -37,6 +37,7 @@ import type { Database } from "../../index";
import { communityBotSyntheticEmail } from "../../../constants";
import { withUniqueDiscriminator } from "../user";
import { nanoid } from "nanoid";
+import { chunk, D1_MAX_IN_PARAMS } from "../_chunk";
// ─── Types ──────────────────────────────────────────────────────────────────
@@ -267,24 +268,33 @@ export async function findWakeCandidates(
if (opts.recipients.length === 0) return [];
const scopeCond = eq(communityReadState.channelId, opts.channelId);
- const rows = await db
- .select({
- botUserId: user.id,
- name: user.name,
- machineId: communityBotBinding.machineId,
- runtime: communityBotBinding.runtime,
- lastReadSeq: communityReadState.lastReadSeq,
- })
- .from(user)
- .innerJoin(communityBotBinding, eq(communityBotBinding.userId, user.id))
- .leftJoin(communityReadState, and(eq(communityReadState.userId, user.id), scopeCond))
- .where(
- and(
- inArray(user.id, opts.recipients),
- eq(user.isBot, true),
- isNull(user.deletedAt)
+ // `recipients` is a message's fan-out audience — unbounded for a large public
+ // channel. Chunk the `inArray` for D1's 100-param limit; no order/limit, and
+ // the seq filter runs in JS below, so concat is loss-free.
+ const rows = (
+ await Promise.all(
+ chunk(opts.recipients, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select({
+ botUserId: user.id,
+ name: user.name,
+ machineId: communityBotBinding.machineId,
+ runtime: communityBotBinding.runtime,
+ lastReadSeq: communityReadState.lastReadSeq,
+ })
+ .from(user)
+ .innerJoin(communityBotBinding, eq(communityBotBinding.userId, user.id))
+ .leftJoin(communityReadState, and(eq(communityReadState.userId, user.id), scopeCond))
+ .where(
+ and(
+ inArray(user.id, ids),
+ eq(user.isBot, true),
+ isNull(user.deletedAt)
+ )
+ )
)
- );
+ )
+ ).flat();
return rows
.filter((r) => (r.lastReadSeq ?? 0) < opts.newSeq)
diff --git a/src/shared/src/db/queries/community/channel.ts b/src/shared/src/db/queries/community/channel.ts
index 2e1691140..c1687a5cd 100644
--- a/src/shared/src/db/queries/community/channel.ts
+++ b/src/shared/src/db/queries/community/channel.ts
@@ -9,6 +9,7 @@ import {
import type { Database } from "../../index";
import { createLogger } from "../../../logger";
import { canManageServer, canSeePrivateChannel } from "../../../utils/community-roles";
+import { chunk, D1_MAX_IN_PARAMS } from "../_chunk";
// Module-level logger — one tag per shared query module.
const log = createLogger({ service: "community-queries" });
@@ -444,10 +445,19 @@ export async function getServersLastActivity(
export async function getChannelsByIds(db: Database, channelIds: string[]) {
if (channelIds.length === 0) return [];
- const rows = await db
- .select(CHANNEL_COLUMNS)
- .from(communityChannel)
- .where(inArray(communityChannel.id, channelIds));
+ // Hydration on the mentions/unreads paths passes up to a full page (200) of
+ // distinct channel ids — chunk the `inArray` for D1's 100-param limit; no
+ // order/limit → concat.
+ const rows = (
+ await Promise.all(
+ chunk(channelIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select(CHANNEL_COLUMNS)
+ .from(communityChannel)
+ .where(inArray(communityChannel.id, ids))
+ )
+ )
+ ).flat();
return rows.map(mapChannelRow);
}
@@ -821,10 +831,11 @@ export async function listVisibleChannelIds(
* channel; posts/threads inherit their parent). Admins get NO special
* visibility — same rule as everyone.
*
- * Bound-parameter ceiling (accepted risk): a viewer across many large servers
- * can produce a big id set; feeding it whole into a downstream `inArray`
- * approaches SQLite's bound-param limit. Same unchunked pattern as
- * `searchMessagesInServer`. Chunk only if it proves a real limit in practice.
+ * Bound-parameter note: a viewer across many large servers can produce a big
+ * id set. Downstream consumers that feed it into an `inArray` (inbox unread,
+ * mentions, search, mark-all-read, agent-inbox) now CHUNK that `inArray` via
+ * `_chunk.ts` so no single statement exceeds D1's 100-param limit — this id set
+ * is safe to return whole.
*/
export async function listVisibleChannelIdsForUser(
db: Database,
diff --git a/src/shared/src/db/queries/community/inbox.ts b/src/shared/src/db/queries/community/inbox.ts
index 3aea414e1..25ca37a09 100644
--- a/src/shared/src/db/queries/community/inbox.ts
+++ b/src/shared/src/db/queries/community/inbox.ts
@@ -1,4 +1,5 @@
import { and, eq, isNotNull, isNull, inArray, ne } from "drizzle-orm";
+import { chunk, D1_MAX_IN_PARAMS } from "../_chunk";
import {
communityChannel,
communityChannelMember,
@@ -72,44 +73,54 @@ export async function listUnreadChannels(
// parent-climbing, so a child is present only when its parent is visible.
// Filtering to actually-unread happens in JS via `isChannelUnread`.
if (visibleChannelIds.length === 0) return [];
- const rows = await db
- .select({
- channelId: communityChannel.id,
- channelName: communityChannel.name,
- serverId: communityChannel.serverId,
- serverName: communityServer.name,
- type: communityChannel.type,
- parentChannelId: communityChannel.parentChannelId,
- lastMessageAt: communityChannel.lastMessageAt,
- lastReadAt: communityReadState.lastReadAt,
- archived: communityChannel.archived,
- // Sidebar / inbox unread badges must ignore messages posted before
- // the viewer joined — otherwise every non-empty channel lights up
- // on first join. `joinedAt` is `notNull()` in the schema and the
- // INNER JOIN below scopes to real member rows, so it's always
- // present. See `isChannelUnread` above.
- joinedAt: communityServerMember.joinedAt,
- })
- .from(communityServerMember)
- .innerJoin(
- communityChannel,
- eq(communityChannel.serverId, communityServerMember.serverId)
- )
- .innerJoin(communityServer, eq(communityServer.id, communityChannel.serverId))
- .leftJoin(
- communityReadState,
- and(
- eq(communityReadState.channelId, communityChannel.id),
- eq(communityReadState.userId, userId)
+ // `visibleChannelIds` is an unbounded authorization scope (all channels the
+ // viewer can see across every server). Chunk the `inArray` for D1's 100-param
+ // limit; there's no order/limit/aggregate, so concat is loss-free (the unread
+ // filtering runs in JS below).
+ const rows = (
+ await Promise.all(
+ chunk(visibleChannelIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select({
+ channelId: communityChannel.id,
+ channelName: communityChannel.name,
+ serverId: communityChannel.serverId,
+ serverName: communityServer.name,
+ type: communityChannel.type,
+ parentChannelId: communityChannel.parentChannelId,
+ lastMessageAt: communityChannel.lastMessageAt,
+ lastReadAt: communityReadState.lastReadAt,
+ archived: communityChannel.archived,
+ // Sidebar / inbox unread badges must ignore messages posted before
+ // the viewer joined — otherwise every non-empty channel lights up
+ // on first join. `joinedAt` is `notNull()` in the schema and the
+ // INNER JOIN below scopes to real member rows, so it's always
+ // present. See `isChannelUnread` above.
+ joinedAt: communityServerMember.joinedAt,
+ })
+ .from(communityServerMember)
+ .innerJoin(
+ communityChannel,
+ eq(communityChannel.serverId, communityServerMember.serverId)
+ )
+ .innerJoin(communityServer, eq(communityServer.id, communityChannel.serverId))
+ .leftJoin(
+ communityReadState,
+ and(
+ eq(communityReadState.channelId, communityChannel.id),
+ eq(communityReadState.userId, userId)
+ )
+ )
+ .where(
+ and(
+ eq(communityServerMember.userId, userId),
+ inArray(communityChannel.id, ids),
+ isNotNull(communityChannel.lastMessageAt)
+ )
+ )
)
)
- .where(
- and(
- eq(communityServerMember.userId, userId),
- inArray(communityChannel.id, visibleChannelIds),
- isNotNull(communityChannel.lastMessageAt)
- )
- );
+ ).flat();
const unread = rows.filter((r) =>
isChannelUnread({
diff --git a/src/shared/src/db/queries/community/mention.ts b/src/shared/src/db/queries/community/mention.ts
index 8c909a131..ef225ff6c 100644
--- a/src/shared/src/db/queries/community/mention.ts
+++ b/src/shared/src/db/queries/community/mention.ts
@@ -2,6 +2,11 @@ import { eq, and, inArray, desc } from "drizzle-orm";
import { communityMention, communityMessage } from "../../community-schema";
import { user } from "../../schema";
import type { Database } from "../../index";
+import { chunk, maxRowsPerInsert, D1_MAX_IN_PARAMS } from "../_chunk";
+
+// communityMention emits 5 bind params/row (id $defaultFn, message_id, user_id,
+// kind default, read default), so a single INSERT caps at floor(100/5)=20 rows.
+const MENTION_INSERT_MAX_ROWS = maxRowsPerInsert(5);
export async function createMentions(
db: Database,
@@ -10,16 +15,24 @@ export async function createMentions(
if (data.userIds.length === 0) return [];
const kind = data.kind ?? "mention";
- const rows = await db
- .insert(communityMention)
- .values(
- data.userIds.map((userId) => ({
- messageId: data.messageId,
- userId,
- kind,
- }))
+ // `@everyone` expands userIds to the whole server roster — unbounded. Chunk so
+ // no INSERT statement exceeds D1's 100-param limit; concat the returned rows.
+ const rows = (
+ await Promise.all(
+ chunk(data.userIds, MENTION_INSERT_MAX_ROWS).map((batch) =>
+ db
+ .insert(communityMention)
+ .values(
+ batch.map((userId) => ({
+ messageId: data.messageId,
+ userId,
+ kind,
+ }))
+ )
+ .returning()
+ )
)
- .returning();
+ ).flat();
return rows;
}
@@ -41,33 +54,54 @@ export async function listUnreadMentions(
visibleChannelIds?: string[];
} = {}
) {
- const conditions = [
+ const baseConditions = [
eq(communityMention.userId, userId),
eq(communityMention.read, 0),
];
- if (opts.kind) conditions.push(eq(communityMention.kind, opts.kind));
- if (opts.visibleChannelIds) {
- conditions.push(inArray(communityMessage.channelId, opts.visibleChannelIds));
+ if (opts.kind) baseConditions.push(eq(communityMention.kind, opts.kind));
+
+ const runQuery = (extra: ReturnType<typeof inArray>[] = []) => {
+ const q = db
+ .select({
+ mention: communityMention,
+ message: communityMessage,
+ author: user,
+ })
+ .from(communityMention)
+ .innerJoin(
+ communityMessage,
+ eq(communityMention.messageId, communityMessage.id)
+ )
+ .innerJoin(user, eq(communityMessage.authorId, user.id))
+ .where(and(...baseConditions, ...extra))
+ // Deterministic truncation: combining mention + reply kinds under a limit
+ // needs an explicit order so the newest mentions win the cap.
+ .orderBy(desc(communityMessage.createdAt));
+ return opts.limit !== undefined ? q.limit(opts.limit) : q;
+ };
+
+ // No channel scoping → single query (agent single-kind read path).
+ if (!opts.visibleChannelIds) {
+ return runQuery();
}
+ // `NULL IN (...)` is never true, so an empty scope yields nothing — match
+ // that instead of running an `inArray(col, [])` (which some drivers reject).
+ if (opts.visibleChannelIds.length === 0) return [];
- const q = db
- .select({
- mention: communityMention,
- message: communityMessage,
- author: user,
- })
- .from(communityMention)
- .innerJoin(
- communityMessage,
- eq(communityMention.messageId, communityMessage.id)
+ // D1 caps a statement at 100 bound params. `visibleChannelIds` is an
+ // unbounded authorization scope (all channels the viewer can see across every
+ // server), so chunk the `inArray` and merge: each chunk runs the SAME
+ // order+limit, then we re-sort by createdAt desc and re-apply the limit so the
+ // global newest wins (not each chunk's newest). `message.createdAt` is in the
+ // projection, so the JS re-sort is exact.
+ const chunks = chunk(opts.visibleChannelIds, D1_MAX_IN_PARAMS);
+ const merged = (
+ await Promise.all(
+ chunks.map((ids) => runQuery([inArray(communityMessage.channelId, ids)]))
)
- .innerJoin(user, eq(communityMessage.authorId, user.id))
- .where(and(...conditions))
- // Deterministic truncation: combining mention + reply kinds under a limit
- // needs an explicit order so the newest mentions win the cap.
- .orderBy(desc(communityMessage.createdAt));
-
- return opts.limit !== undefined ? q.limit(opts.limit) : q;
+ ).flat();
+ merged.sort((a, b) => (a.message.createdAt < b.message.createdAt ? 1 : -1));
+ return opts.limit !== undefined ? merged.slice(0, opts.limit) : merged;
}
export async function markMentionsRead(
diff --git a/src/shared/src/db/queries/community/message.ts b/src/shared/src/db/queries/community/message.ts
index 535342447..96af064c5 100644
--- a/src/shared/src/db/queries/community/message.ts
+++ b/src/shared/src/db/queries/community/message.ts
@@ -9,6 +9,7 @@ import {
import { user } from "../../schema";
import type { Database } from "../../index";
import { createLogger } from "../../../logger";
+import { chunk, D1_MAX_IN_PARAMS } from "../_chunk";
/**
* Atomically claim the next seq value for a channel. A single top-level UPSERT
@@ -582,30 +583,40 @@ export async function getLatestMessagesByChannelIds(
): Promise<Array<{ channelId: string; id: string; createdAt: string }>> {
if (channelIds.length === 0) return [];
- const latestDates = db
- .select({
- channelId: communityMessage.channelId,
- maxCreatedAt: sql<string>`MAX(${communityMessage.createdAt})`.as("max_created_at"),
- })
- .from(communityMessage)
- .where(inArray(communityMessage.channelId, channelIds))
- .groupBy(communityMessage.channelId)
- .as("latest_dates");
+ // D1 caps a statement at 100 bound params. Chunk the `inArray` subquery — each
+ // channel id lands in exactly one chunk (chunks partition), so the per-channel
+ // GROUP BY MAX is complete within its chunk; concat + the existing per-channel
+ // dedup below merges losslessly.
+ const runChunk = (ids: string[]) => {
+ const latestDates = db
+ .select({
+ channelId: communityMessage.channelId,
+ maxCreatedAt: sql<string>`MAX(${communityMessage.createdAt})`.as("max_created_at"),
+ })
+ .from(communityMessage)
+ .where(inArray(communityMessage.channelId, ids))
+ .groupBy(communityMessage.channelId)
+ .as("latest_dates");
+
+ return db
+ .select({
+ channelId: communityMessage.channelId,
+ id: communityMessage.id,
+ createdAt: communityMessage.createdAt,
+ })
+ .from(communityMessage)
+ .innerJoin(
+ latestDates,
+ and(
+ eq(communityMessage.channelId, latestDates.channelId),
+ eq(communityMessage.createdAt, latestDates.maxCreatedAt)
+ )
+ );
+ };
- const rows = await db
- .select({
- channelId: communityMessage.channelId,
- id: communityMessage.id,
- createdAt: communityMessage.createdAt,
- })
- .from(communityMessage)
- .innerJoin(
- latestDates,
- and(
- eq(communityMessage.channelId, latestDates.channelId),
- eq(communityMessage.createdAt, latestDates.maxCreatedAt)
- )
- );
+ const rows = (
+ await Promise.all(chunk(channelIds, D1_MAX_IN_PARAMS).map(runChunk))
+ ).flat();
// Deduplicate on channelId: two messages in the same channel could share an
// exact `createdAt` (millisecond collisions on batched inserts). Pick the
@@ -628,30 +639,38 @@ export async function getLatestMessagesByChannelIds(
export async function getFirstMessageByChannelIds(db: Database, channelIds: string[]) {
if (channelIds.length === 0) return [];
- // Use a subquery to get the min createdAt per channel, then join to get the content
- const firstDates = db
- .select({
- channelId: communityMessage.channelId,
- minCreatedAt: sql<string>`MIN(${communityMessage.createdAt})`.as("min_created_at"),
- })
- .from(communityMessage)
- .where(inArray(communityMessage.channelId, channelIds))
- .groupBy(communityMessage.channelId)
- .as("first_dates");
+ // Use a subquery to get the min createdAt per channel, then join to get the
+ // content. Chunk the `inArray` for D1's 100-param limit — GROUP BY channelId
+ // partitions cleanly across chunks (defensive; input is page-bounded today).
+ const runChunk = (ids: string[]) => {
+ const firstDates = db
+ .select({
+ channelId: communityMessage.channelId,
+ minCreatedAt: sql<string>`MIN(${communityMessage.createdAt})`.as("min_created_at"),
+ })
+ .from(communityMessage)
+ .where(inArray(communityMessage.channelId, ids))
+ .groupBy(communityMessage.channelId)
+ .as("first_dates");
+
+ return db
+ .select({
+ channelId: communityMessage.channelId,
+ content: communityMessage.content,
+ })
+ .from(communityMessage)
+ .innerJoin(
+ firstDates,
+ and(
+ eq(communityMessage.channelId, firstDates.channelId),
+ eq(communityMessage.createdAt, firstDates.minCreatedAt)
+ )
+ );
+ };
- const rows = await db
- .select({
- channelId: communityMessage.channelId,
- content: communityMessage.content,
- })
- .from(communityMessage)
- .innerJoin(
- firstDates,
- and(
- eq(communityMessage.channelId, firstDates.channelId),
- eq(communityMessage.createdAt, firstDates.minCreatedAt)
- )
- );
+ const rows = (
+ await Promise.all(chunk(channelIds, D1_MAX_IN_PARAMS).map(runChunk))
+ ).flat();
// Deduplicate in case of exact same createdAt within a channel
const seen = new Set<string>();
diff --git a/src/shared/src/db/queries/community/read-state.ts b/src/shared/src/db/queries/community/read-state.ts
index 31ea98b0a..4e76ef15f 100644
--- a/src/shared/src/db/queries/community/read-state.ts
+++ b/src/shared/src/db/queries/community/read-state.ts
@@ -1,6 +1,7 @@
import { eq, and, inArray, lt, sql } from "drizzle-orm";
import { communityReadState } from "../../community-schema";
import type { Database } from "../../index";
+import { chunk, maxRowsPerInsert, D1_MAX_IN_PARAMS } from "../_chunk";
import {
getLatestMessagesByChannelIds,
getMessageByChannelAndSeq,
@@ -163,21 +164,28 @@ export async function markAllServerChannelsRead(
// fires per statement; we can't fold every channel into a single insert
// with `onConflictDoUpdate` because each channel has a DIFFERENT
// `(lastReadAt, lastReadMessageId)` pair.
- const existing = await db
- .select({
- id: communityReadState.id,
- channelId: communityReadState.channelId,
- })
- .from(communityReadState)
- .where(
- and(
- eq(communityReadState.userId, userId),
- inArray(
- communityReadState.channelId,
- latest.map((l) => l.channelId)
- )
+ // `latest` ranges over channels-with-messages (a subset of the visible set,
+ // still possibly >100). Chunk the `inArray` for D1's 100-param limit; no
+ // order/limit → concat.
+ const latestChannelIds = latest.map((l) => l.channelId);
+ const existing = (
+ await Promise.all(
+ chunk(latestChannelIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select({
+ id: communityReadState.id,
+ channelId: communityReadState.channelId,
+ })
+ .from(communityReadState)
+ .where(
+ and(
+ eq(communityReadState.userId, userId),
+ inArray(communityReadState.channelId, ids)
+ )
+ )
)
- );
+ )
+ ).flat();
const existingByChannel = new Map<string, string>();
for (const row of existing) {
@@ -220,14 +228,19 @@ export async function markAllServerChannelsRead(
}
if (toInsert.length > 0) {
- await db.insert(communityReadState).values(
- toInsert.map((i) => ({
- userId,
- channelId: i.channelId,
- lastReadAt: i.createdAt,
- lastReadMessageId: i.msgId,
- }))
- );
+ // communityReadState emits 6 bind params/row (id $defaultFn, user_id,
+ // channel_id, last_read_at, last_read_message_id, last_read_seq default),
+ // so cap at floor(100/6)=16 rows/statement for D1's 100-param limit.
+ for (const batch of chunk(toInsert, maxRowsPerInsert(6))) {
+ await db.insert(communityReadState).values(
+ batch.map((i) => ({
+ userId,
+ channelId: i.channelId,
+ lastReadAt: i.createdAt,
+ lastReadMessageId: i.msgId,
+ }))
+ );
+ }
}
return latest.length;
diff --git a/src/shared/src/db/queries/community/search.ts b/src/shared/src/db/queries/community/search.ts
index 30f07c849..3508eb831 100644
--- a/src/shared/src/db/queries/community/search.ts
+++ b/src/shared/src/db/queries/community/search.ts
@@ -3,6 +3,7 @@ import { communityMessage, communityChannel } from "../../community-schema";
import { user } from "../../schema";
import type { Database } from "../../index";
import { escapeLikePattern } from "../../../utils/sql-like";
+import { chunk, D1_MAX_IN_PARAMS } from "../_chunk";
const DEFAULT_LIMIT = 50;
@@ -77,25 +78,40 @@ export async function searchMessagesInServer(
if (opts.visibleChannelIds && opts.visibleChannelIds.length === 0) return [];
const pattern = `%${escapeLikePattern(opts.query)}%`;
- const conditions = [
+ const baseConditions = [
sql`${communityMessage.content} LIKE ${pattern} ESCAPE '\\'`,
eq(communityChannel.serverId, opts.serverId),
];
- if (opts.visibleChannelIds) {
- conditions.push(inArray(communityMessage.channelId, opts.visibleChannelIds));
- }
- return db
- .select({
- message: communityMessage,
- author: user,
- })
- .from(communityMessage)
- .innerJoin(user, eq(communityMessage.authorId, user.id))
- .innerJoin(
- communityChannel,
- eq(communityMessage.channelId, communityChannel.id)
+ const runQuery = (extra: ReturnType<typeof inArray>[] = []) =>
+ db
+ .select({
+ message: communityMessage,
+ author: user,
+ })
+ .from(communityMessage)
+ .innerJoin(user, eq(communityMessage.authorId, user.id))
+ .innerJoin(
+ communityChannel,
+ eq(communityMessage.channelId, communityChannel.id)
+ )
+ .where(and(...baseConditions, ...extra))
+ .limit(limit);
+
+ if (!opts.visibleChannelIds) return runQuery();
+
+ // D1 caps a statement at 100 bound params; `visibleChannelIds` is unbounded.
+ // Chunk the `inArray` and merge. There's no ORDER BY contract today, so plain
+ // concat+slice would bias toward early-chunk channels (a match in a late chunk
+ // gets dropped once an early chunk fills `limit`). Sort the merged rows by
+ // createdAt desc before slicing → a defensible "most-recent matches" rule
+ // instead of chunk-order bias. `createdAt` is in the projection.
+ const chunks = chunk(opts.visibleChannelIds, D1_MAX_IN_PARAMS);
+ const merged = (
+ await Promise.all(
+ chunks.map((ids) => runQuery([inArray(communityMessage.channelId, ids)]))
)
- .where(and(...conditions))
- .limit(limit);
+ ).flat();
+ merged.sort((a, b) => (a.message.createdAt < b.message.createdAt ? 1 : -1));
+ return merged.slice(0, limit);
}
diff --git a/src/shared/src/db/queries/community/server-folder.ts b/src/shared/src/db/queries/community/server-folder.ts
index c9023d3e3..3272863a5 100644
--- a/src/shared/src/db/queries/community/server-folder.ts
+++ b/src/shared/src/db/queries/community/server-folder.ts
@@ -5,6 +5,11 @@ import {
communityServer,
} from "../../community-schema";
import type { Database } from "../../index";
+import { chunk, maxRowsPerInsert } from "../_chunk";
+
+// communityServerFolderItem emits 3 bind params/row (folder_id, server_id,
+// position), so a single INSERT caps at floor(100/3)=33 rows for D1's limit.
+const FOLDER_ITEM_INSERT_MAX_ROWS = maxRowsPerInsert(3);
export async function createFolder(
db: Database,
@@ -25,13 +30,7 @@ export async function createFolder(
.returning();
if (data.serverIds && data.serverIds.length > 0) {
- await db.insert(communityServerFolderItem).values(
- data.serverIds.map((serverId, idx) => ({
- folderId: folder!.id,
- serverId,
- position: idx,
- }))
- );
+ await insertFolderItems(db, folder!.id, data.serverIds);
}
return folder!;
@@ -77,13 +76,24 @@ export async function replaceFolderItems(
.where(eq(communityServerFolderItem.folderId, folderId));
if (serverIds.length > 0) {
- await db.insert(communityServerFolderItem).values(
- serverIds.map((serverId, idx) => ({
- folderId,
- serverId,
- position: idx,
- }))
- );
+ await insertFolderItems(db, folderId, serverIds);
+ }
+}
+
+// Bulk-insert folder items, chunked so no INSERT exceeds D1's 100-param limit.
+// `position` is the item's index in the FULL list, preserved across chunks.
+async function insertFolderItems(
+ db: Database,
+ folderId: string,
+ serverIds: string[]
+) {
+ const items = serverIds.map((serverId, idx) => ({
+ folderId,
+ serverId,
+ position: idx,
+ }));
+ for (const batch of chunk(items, FOLDER_ITEM_INSERT_MAX_ROWS)) {
+ await db.insert(communityServerFolderItem).values(batch);
}
}
diff --git a/src/shared/src/db/queries/community/server.ts b/src/shared/src/db/queries/community/server.ts
index 5af0677ef..2f1675b00 100644
--- a/src/shared/src/db/queries/community/server.ts
+++ b/src/shared/src/db/queries/community/server.ts
@@ -9,6 +9,7 @@ import {
} from "../../community-schema";
import { user } from "../../schema";
import type { Database } from "../../index";
+import { chunk, D1_MAX_IN_PARAMS } from "../_chunk";
export async function createServer(
db: Database,
@@ -217,7 +218,15 @@ export async function resolveServerByNameForMember(
export async function getServersByIds(db: Database, serverIds: string[]) {
if (serverIds.length === 0) return [];
- return db.select().from(communityServer).where(inArray(communityServer.id, serverIds));
+ // Chunk the `inArray` for D1's 100-param limit (mentions hydration can pass a
+ // page's worth of distinct server ids); no order/limit → concat.
+ return (
+ await Promise.all(
+ chunk(serverIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db.select().from(communityServer).where(inArray(communityServer.id, ids))
+ )
+ )
+ ).flat();
}
export async function setServerIcon(
diff --git a/src/shared/src/db/queries/community/thread.ts b/src/shared/src/db/queries/community/thread.ts
index eb83bf912..158291cae 100644
--- a/src/shared/src/db/queries/community/thread.ts
+++ b/src/shared/src/db/queries/community/thread.ts
@@ -2,6 +2,7 @@ import { and, eq, inArray } from "drizzle-orm";
import { communityChannel, communityChannelMember } from "../../community-schema";
import { user } from "../../schema";
import type { Database } from "../../index";
+import { chunk, maxRowsPerInsert, D1_MAX_IN_PARAMS } from "../_chunk";
// The NOTIFICATION set for a thread OR forum_post — now relation='notify' rows
// on `community_channel_member` (formerly the standalone
@@ -52,17 +53,23 @@ export async function addThreadParticipants(
rows: { userId: string; source: ThreadParticipantSource }[]
) {
if (rows.length === 0) return;
- await db
- .insert(communityChannelMember)
- .values(
- rows.map((r) => ({
- channelId: threadChannelId,
- userId: r.userId,
- relation: "notify",
- source: r.source,
- }))
- )
- .onConflictDoNothing({ target: [...NOTIFY_CONFLICT_TARGET] });
+ // communityChannelMember emits 6 bind params/row (id $defaultFn, channel_id,
+ // user_id, relation, source, added_at $defaultFn; added_by is an unsupplied
+ // literal null, not a param), so cap at floor(100/6)=16 rows for D1's 100-param
+ // limit. `onConflictDoNothing` adds no VALUES params.
+ for (const batch of chunk(rows, maxRowsPerInsert(6))) {
+ await db
+ .insert(communityChannelMember)
+ .values(
+ batch.map((r) => ({
+ channelId: threadChannelId,
+ userId: r.userId,
+ relation: "notify",
+ source: r.source,
+ }))
+ )
+ .onConflictDoNothing({ target: [...NOTIFY_CONFLICT_TARGET] });
+ }
}
// The NOTIFY set: every participant userId. This is what thread fan-out /
@@ -210,15 +217,22 @@ export async function listParticipatingThreadIds(
userId: string
): Promise<string[]> {
if (threadChannelIds.length === 0) return [];
- const rows = await db
- .select({ channelId: communityChannelMember.channelId })
- .from(communityChannelMember)
- .where(
- and(
- inArray(communityChannelMember.channelId, threadChannelIds),
- eq(communityChannelMember.userId, userId),
- eq(communityChannelMember.relation, "notify")
+ // Chunk the `inArray` for D1's 100-param limit; no order/limit → concat.
+ const rows = (
+ await Promise.all(
+ chunk(threadChannelIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select({ channelId: communityChannelMember.channelId })
+ .from(communityChannelMember)
+ .where(
+ and(
+ inArray(communityChannelMember.channelId, ids),
+ eq(communityChannelMember.userId, userId),
+ eq(communityChannelMember.relation, "notify")
+ )
+ )
)
- );
+ )
+ ).flat();
return rows.map((r) => r.channelId);
}
diff --git a/src/shared/src/db/queries/task.ts b/src/shared/src/db/queries/task.ts
index cef43618a..ba7e0346d 100644
--- a/src/shared/src/db/queries/task.ts
+++ b/src/shared/src/db/queries/task.ts
@@ -3,6 +3,7 @@ import { alias } from "drizzle-orm/sqlite-core";
import { agentTaskQueue, taskMessage, conversation } from "../schema";
import type { Database } from "../index";
import { ClaimedTaskRowSchema } from "../../schemas";
+import { chunk, D1_MAX_IN_PARAMS } from "./_chunk";
import { TASK_TYPES } from "../../constants";
export async function createTask(
@@ -106,20 +107,28 @@ export async function findSteerableReplacement(
if (steerableActive.length === 0) return null;
const contextKeys = steerableActive.map((t) => t.contextKey!);
- const queuedWithMatchingKeys = await db
- .select({
- conversationId: agentTaskQueue.conversationId,
- contextKey: agentTaskQueue.contextKey,
- })
- .from(agentTaskQueue)
- .where(
- and(
- eq(agentTaskQueue.agentId, agentId),
- eq(agentTaskQueue.workspaceId, workspaceId),
- eq(agentTaskQueue.status, "queued"),
- inArray(agentTaskQueue.contextKey, contextKeys)
+ // Chunk the `inArray` for D1's 100-param limit; the result feeds a JS
+ // membership set below, so concat across chunks is loss-free.
+ const queuedWithMatchingKeys = (
+ await Promise.all(
+ chunk(contextKeys, D1_MAX_IN_PARAMS).map((keys) =>
+ db
+ .select({
+ conversationId: agentTaskQueue.conversationId,
+ contextKey: agentTaskQueue.contextKey,
+ })
+ .from(agentTaskQueue)
+ .where(
+ and(
+ eq(agentTaskQueue.agentId, agentId),
+ eq(agentTaskQueue.workspaceId, workspaceId),
+ eq(agentTaskQueue.status, "queued"),
+ inArray(agentTaskQueue.contextKey, keys)
+ )
+ )
)
- );
+ )
+ ).flat();
const queuedSet = new Set(queuedWithMatchingKeys.map((q) => `${q.conversationId}:${q.contextKey}`));
for (const active of steerableActive) {
@@ -187,14 +196,16 @@ export async function claimTask(db: Database, agentId: string, workspaceId: stri
let candidates = await candidateQuery;
- // If no non-steerable candidate, try steerable conversations (batched, capped at 50)
+ // If no non-steerable candidate, try steerable conversations (capped at 50).
+ // Match on a synthesized composite key `conversationId || ':' || contextKey`
+ // via a single `inArray` instead of 50 `or(and(eq,eq))` legs. The old shape
+ // bound 2 params/leg → 50 legs = 100 params, plus the agentId/workspaceId/
+ // status params = ~103, over D1's 100 limit. The composite key binds 1
+ // param/leg → 50, well under. Key format mirrors `queuedSet` above.
if (candidates.length === 0 && steerableConvContextKeys.size > 0) {
- const steerConditions = [...steerableConvContextKeys.entries()].slice(0, 50).map(([convId, ctxKey]) =>
- and(
- eq(agentTaskQueue.conversationId, convId),
- eq(agentTaskQueue.contextKey, ctxKey),
- )
- );
+ const steerKeys = [...steerableConvContextKeys.entries()]
+ .slice(0, 50)
+ .map(([convId, ctxKey]) => `${convId}:${ctxKey}`);
const steerCandidates = await db
.select({ id: agentTaskQueue.id })
.from(agentTaskQueue)
@@ -203,7 +214,10 @@ export async function claimTask(db: Database, agentId: string, workspaceId: stri
eq(agentTaskQueue.agentId, agentId),
eq(agentTaskQueue.workspaceId, workspaceId),
eq(agentTaskQueue.status, "queued"),
- or(...steerConditions),
+ inArray(
+ sql`${agentTaskQueue.conversationId} || ':' || ${agentTaskQueue.contextKey}`,
+ steerKeys
+ ),
)
)
.orderBy(desc(agentTaskQueue.priority), asc(agentTaskQueue.createdAt))
diff --git a/src/shared/src/db/queries/workspace-file-request.ts b/src/shared/src/db/queries/workspace-file-request.ts
index f900fb062..7ae4995c8 100644
--- a/src/shared/src/db/queries/workspace-file-request.ts
+++ b/src/shared/src/db/queries/workspace-file-request.ts
@@ -1,6 +1,7 @@
import { eq, and, inArray, lt } from "drizzle-orm";
import { workspaceFileRequest } from "../schema";
import type { Database } from "../index";
+import { chunk, D1_MAX_IN_PARAMS } from "./_chunk";
export async function createRequest(
db: Database,
@@ -35,10 +36,16 @@ export async function getPendingByWorkspace(
export async function markDispatched(db: Database, ids: string[]) {
if (ids.length === 0) return;
- await db
- .update(workspaceFileRequest)
- .set({ status: "dispatched", updatedAt: new Date().toISOString() })
- .where(inArray(workspaceFileRequest.id, ids));
+ // `ids` come from `getPendingByWorkspace` (unbounded SELECT, no LIMIT). Chunk
+ // the `inArray` for D1's 100-param limit — the UPDATE also binds status +
+ // updatedAt, so D1_MAX_IN_PARAMS (90) leaves headroom.
+ const now = new Date().toISOString();
+ for (const batch of chunk(ids, D1_MAX_IN_PARAMS)) {
+ await db
+ .update(workspaceFileRequest)
+ .set({ status: "dispatched", updatedAt: now })
+ .where(inArray(workspaceFileRequest.id, batch));
+ }
}
export async function completeRequest(
diff --git a/src/shared/test/queries/chunk.test.ts b/src/shared/test/queries/chunk.test.ts
new file mode 100644
index 000000000..a6f54d12b
--- /dev/null
+++ b/src/shared/test/queries/chunk.test.ts
@@ -0,0 +1,70 @@
+import { describe, it, expect } from "vitest";
+import {
+ chunk,
+ maxRowsPerInsert,
+ D1_MAX_BIND_PARAMS,
+ D1_MAX_IN_PARAMS,
+} from "../../src/db/queries/_chunk";
+
+describe("chunk", () => {
+ it("returns [] for an empty array", () => {
+ expect(chunk([], 10)).toEqual([]);
+ });
+
+ it("returns a single chunk when N < size", () => {
+ expect(chunk([1, 2, 3], 10)).toEqual([[1, 2, 3]]);
+ });
+
+ it("returns a single full chunk when N === size", () => {
+ expect(chunk([1, 2, 3], 3)).toEqual([[1, 2, 3]]);
+ });
+
+ it("splits into two chunks when N === size + 1", () => {
+ expect(chunk([1, 2, 3, 4], 3)).toEqual([[1, 2, 3], [4]]);
+ });
+
+ it("splits evenly when N === 2 * size", () => {
+ expect(chunk([1, 2, 3, 4], 2)).toEqual([
+ [1, 2],
+ [3, 4],
+ ]);
+ });
+
+ it("preserves order across chunks", () => {
+ const arr = Array.from({ length: 250 }, (_, i) => i);
+ const chunks = chunk(arr, 90);
+ expect(chunks.map((c) => c.length)).toEqual([90, 90, 70]);
+ expect(chunks.flat()).toEqual(arr);
+ });
+
+ it("throws on size < 1", () => {
+ expect(() => chunk([1], 0)).toThrow();
+ });
+});
+
+describe("maxRowsPerInsert", () => {
+ it("caps so rows*paramsPerRow never exceeds 100", () => {
+ for (const p of [1, 2, 3, 4, 5, 6, 7, 10, 33, 50, 100]) {
+ const rows = maxRowsPerInsert(p);
+ expect(rows * p).toBeLessThanOrEqual(D1_MAX_BIND_PARAMS);
+ expect((rows + 1) * p).toBeGreaterThan(D1_MAX_BIND_PARAMS);
+ }
+ });
+
+ it("matches the pinned per-table caps", () => {
+ expect(maxRowsPerInsert(5)).toBe(20); // communityMention
+ expect(maxRowsPerInsert(6)).toBe(16); // communityReadState, communityChannelMember
+ expect(maxRowsPerInsert(3)).toBe(33); // communityServerFolderItem
+ });
+
+ it("throws on paramsPerRow < 1", () => {
+ expect(() => maxRowsPerInsert(0)).toThrow();
+ });
+});
+
+describe("constants", () => {
+ it("D1_MAX_IN_PARAMS leaves headroom under the hard limit", () => {
+ expect(D1_MAX_BIND_PARAMS).toBe(100);
+ expect(D1_MAX_IN_PARAMS).toBeLessThan(D1_MAX_BIND_PARAMS);
+ });
+});
diff --git a/src/shared/test/queries/param-limit.test.ts b/src/shared/test/queries/param-limit.test.ts
new file mode 100644
index 000000000..ecc887cfd
--- /dev/null
+++ b/src/shared/test/queries/param-limit.test.ts
@@ -0,0 +1,149 @@
+import { describe, it, expect, vi } from "vitest";
+import { drizzle } from "drizzle-orm/d1";
+import {
+ communityMention,
+ communityReadState,
+ communityChannelMember,
+ communityServerFolderItem,
+} from "../../src/db/community-schema";
+import * as mentionQueries from "../../src/db/queries/community/mention";
+import * as attachmentQueries from "../../src/db/queries/community/attachment";
+import { D1_MAX_BIND_PARAMS } from "../../src/db/queries/_chunk";
+
+const fakeDb = drizzle({} as never);
+
+// The REAL number of bind params a multi-row insert emits per row. Drizzle binds
+// every non-disabled column of every row (including a $defaultFn primary key and
+// literal-default columns), so we count params on the actually-built statement
+// rather than trusting the "logical" column count. This locks the caps in
+// _chunk usage against schema drift — add a column, this test forces a re-cap.
+function paramsPerRow(table: Parameters<typeof fakeDb.insert>[0], row: Record<string, unknown>): number {
+ const two = fakeDb
+ .insert(table)
+ .values([row, row])
+ .toSQL();
+ return two.params.length / 2;
+}
+
+describe("real emitted params per row (pins the insert caps)", () => {
+ it("communityMention = 5", () => {
+ const p = paramsPerRow(communityMention, {
+ messageId: "m1",
+ userId: "u1",
+ kind: "mention",
+ });
+ expect(p).toBe(5);
+ });
+
+ it("communityReadState = 6", () => {
+ const p = paramsPerRow(communityReadState, {
+ userId: "u1",
+ channelId: "c1",
+ lastReadAt: "2026-01-01T00:00:00.000Z",
+ lastReadMessageId: "m1",
+ });
+ expect(p).toBe(6);
+ });
+
+ it("communityChannelMember = 6", () => {
+ const p = paramsPerRow(communityChannelMember, {
+ channelId: "c1",
+ userId: "u1",
+ relation: "notify",
+ source: "mention",
+ });
+ expect(p).toBe(6);
+ });
+
+ it("communityServerFolderItem = 3", () => {
+ const p = paramsPerRow(communityServerFolderItem, {
+ folderId: "f1",
+ serverId: "s1",
+ position: 0,
+ });
+ expect(p).toBe(3);
+ });
+});
+
+// Capturing db that records each insert's row count and resolves .returning().
+function makeInsertCapture() {
+ const inserts: number[] = [];
+ const db: any = {
+ insert: vi.fn(() => {
+ const builder: any = {};
+ builder.values = vi.fn((rows: unknown[]) => {
+ inserts.push(rows.length);
+ const chainable: any = {
+ returning: vi.fn(() =>
+ Promise.resolve((rows as Record<string, unknown>[]).map((r, i) => ({ ...r, id: `row-${inserts.length}-${i}` })))
+ ),
+ onConflictDoNothing: vi.fn(() => Promise.resolve()),
+ };
+ return chainable;
+ });
+ return builder;
+ }),
+ };
+ return { db, inserts };
+}
+
+describe("createMentions chunking", () => {
+ it("100 userIds → 5 insert statements of ≤20 rows, returns all 100", async () => {
+ const { db, inserts } = makeInsertCapture();
+ const userIds = Array.from({ length: 100 }, (_, i) => `u${i}`);
+ const rows = await mentionQueries.createMentions(db, {
+ messageId: "m1",
+ userIds,
+ });
+ // 5 params/row → cap 20 → 100/20 = 5 statements.
+ expect(inserts).toEqual([20, 20, 20, 20, 20]);
+ for (const n of inserts) expect(n * 5).toBeLessThanOrEqual(D1_MAX_BIND_PARAMS);
+ expect(rows).toHaveLength(100);
+ });
+
+ it("empty userIds → no insert", async () => {
+ const { db, inserts } = makeInsertCapture();
+ const rows = await mentionQueries.createMentions(db, { messageId: "m1", userIds: [] });
+ expect(inserts).toEqual([]);
+ expect(rows).toEqual([]);
+ });
+
+ it("21 userIds → 2 statements (20 + 1)", async () => {
+ const { db, inserts } = makeInsertCapture();
+ const userIds = Array.from({ length: 21 }, (_, i) => `u${i}`);
+ await mentionQueries.createMentions(db, { messageId: "m1", userIds });
+ expect(inserts).toEqual([20, 1]);
+ });
+});
+
+// Capturing db for a chunked SELECT that ends in .orderBy(). Returns rows by
+// call index (chunk order is deterministic: first chunk = ids[0..89], etc.).
+function makeSelectCapture(rowsPerCall: unknown[][]) {
+ let call = 0;
+ const db: any = {
+ select: vi.fn(() => {
+ const chain: any = {};
+ chain.from = vi.fn(() => chain);
+ chain.where = vi.fn(() => chain);
+ chain.orderBy = vi.fn(() => Promise.resolve(rowsPerCall[call++] ?? []));
+ return chain;
+ }),
+ };
+ return { db, callCount: () => call };
+}
+
+describe("listByMessageIds chunking + global re-sort", () => {
+ it("150 ids → 2 chunks, rows re-sorted globally by (position, createdAt)", async () => {
+ // First chunk returns position 2; second chunk returns position 1. A correct
+ // global re-sort must put position 1 first even though its chunk ran second
+ // (per-chunk order alone would be wrong).
+ const { db, callCount } = makeSelectCapture([
+ [{ messageId: "m0", position: 2, createdAt: "2026-01-01T00:00:02.000Z" }],
+ [{ messageId: "m100", position: 1, createdAt: "2026-01-01T00:00:01.000Z" }],
+ ]);
+ const ids = Array.from({ length: 150 }, (_, i) => `m${i}`);
+ const rows = await attachmentQueries.listByMessageIds(db, ids);
+ expect(callCount()).toBe(2); // 150 ids → 2 chunks (90 + 60)
+ expect(rows.map((r: any) => r.position)).toEqual([1, 2]);
+ });
+});