Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(shared): chunk the remaining inbox/message hydration inArrays

Gener2242d30a9f85
diff --git a/src/shared/src/db/queries/community/agent-inbox.ts b/src/shared/src/db/queries/community/agent-inbox.ts
index 5525e8d5c..3ef97b814 100644
--- a/src/shared/src/db/queries/community/agent-inbox.ts
+++ b/src/shared/src/db/queries/community/agent-inbox.ts
@@ -78,34 +78,48 @@ async function resolveScopeRefs(
const channelIds = [...new Set(scopes.map((s) => s.channelId))];
if (channelIds.length === 0) return new Map();
- const channels = await db
- .select({
- id: communityChannel.id,
- name: communityChannel.name,
- type: communityChannel.type,
- serverId: communityChannel.serverId,
- parentChannelId: communityChannel.parentChannelId,
- parentMessageId: communityChannel.parentMessageId,
- })
- .from(communityChannel)
- .where(inArray(communityChannel.id, channelIds));
+ // Chunk the `inArray` for D1's 100-param limit — a page of up to 201 agent
+ // messages can span >100 distinct channels; no order/limit → concat.
+ const channels = (
+ await Promise.all(
+ chunk(channelIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select({
+ id: communityChannel.id,
+ name: communityChannel.name,
+ type: communityChannel.type,
+ serverId: communityChannel.serverId,
+ parentChannelId: communityChannel.parentChannelId,
+ parentMessageId: communityChannel.parentMessageId,
+ })
+ .from(communityChannel)
+ .where(inArray(communityChannel.id, ids))
+ )
+ )
+ ).flat();
const dmChannelIds = channels.filter((c) => c.type === "dm").map((c) => c.id);
// DM peers: the OTHER access member per DM channel, resolved to a handle.
const dmMemberRows = dmChannelIds.length
- ? await db
- .select({
- channelId: communityChannelMember.channelId,
- userId: communityChannelMember.userId,
- })
- .from(communityChannelMember)
- .where(
- and(
- inArray(communityChannelMember.channelId, dmChannelIds),
- eq(communityChannelMember.relation, "access"),
- ne(communityChannelMember.userId, viewerId)
+ ? (
+ await Promise.all(
+ chunk(dmChannelIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select({
+ channelId: communityChannelMember.channelId,
+ userId: communityChannelMember.userId,
+ })
+ .from(communityChannelMember)
+ .where(
+ and(
+ inArray(communityChannelMember.channelId, ids),
+ eq(communityChannelMember.relation, "access"),
+ ne(communityChannelMember.userId, viewerId)
+ )
+ )
+ )
)
- )
+ ).flat()
: [];
const peerByDmChannel = new Map<string, string>();
for (const m of dmMemberRows) {
@@ -113,10 +127,16 @@ async function resolveScopeRefs(
}
const dmPeerIds = [...new Set([...peerByDmChannel.values()])];
const dmPeerUsers = dmPeerIds.length
- ? await db
- .select({ id: user.id, name: user.name, discriminator: user.discriminator })
- .from(user)
- .where(inArray(user.id, dmPeerIds))
+ ? (
+ await Promise.all(
+ chunk(dmPeerIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select({ id: user.id, name: user.name, discriminator: user.discriminator })
+ .from(user)
+ .where(inArray(user.id, ids))
+ )
+ )
+ ).flat()
: [];
const dmPeerById = new Map(dmPeerUsers.map((u) => [u.id, u]));
@@ -128,19 +148,29 @@ async function resolveScopeRefs(
];
const serverIds = [...new Set(channels.map((c) => c.serverId).filter((x): x is string => !!x))];
+ // Each of these `inArray`s is fed by an unbounded channel set (a page of up to
+ // 201 agent messages), so chunk for D1's 100-param limit; no order/limit → concat.
+ const chunkedIn = <T>(
+ ids: string[],
+ run: (batch: string[]) => Promise<T[]>
+ ): Promise<T[]> =>
+ ids.length
+ ? Promise.all(chunk(ids, D1_MAX_IN_PARAMS).map(run)).then((r) => r.flat())
+ : Promise.resolve([]);
+
const [parentChannels, servers, parentMessages] = await Promise.all([
- parentChannelIds.length
- ? db
+ chunkedIn(parentChannelIds, (ids) =>
+ db
.select({ id: communityChannel.id, name: communityChannel.name })
.from(communityChannel)
- .where(inArray(communityChannel.id, parentChannelIds))
- : Promise.resolve([]),
- serverIds.length
- ? db.select({ id: communityServer.id, name: communityServer.name }).from(communityServer).where(inArray(communityServer.id, serverIds))
- : Promise.resolve([]),
- parentMessageIds.length
- ? db.select({ id: communityMessage.id, seq: communityMessage.seq }).from(communityMessage).where(inArray(communityMessage.id, parentMessageIds))
- : Promise.resolve([]),
+ .where(inArray(communityChannel.id, ids))
+ ),
+ chunkedIn(serverIds, (ids) =>
+ db.select({ id: communityServer.id, name: communityServer.name }).from(communityServer).where(inArray(communityServer.id, ids))
+ ),
+ chunkedIn(parentMessageIds, (ids) =>
+ db.select({ id: communityMessage.id, seq: communityMessage.seq }).from(communityMessage).where(inArray(communityMessage.id, ids))
+ ),
]);
const parentChannelById = new Map(parentChannels.map((c) => [c.id, c]));
@@ -198,12 +228,19 @@ export async function toAgentMessages(
): Promise<Message[]> {
if (rows.length === 0) return [];
+ const authorIds = [...new Set(rows.map((r) => r.authorId))];
const [refs, users, replyByMessageId] = await Promise.all([
resolveScopeRefs(db, rows, viewerId),
- db
- .select({ id: user.id, name: user.name, discriminator: user.discriminator })
- .from(user)
- .where(inArray(user.id, [...new Set(rows.map((r) => r.authorId))])),
+ // Chunk the `inArray` for D1's 100-param limit — a page of up to 201 agent
+ // messages can carry >100 distinct authors; no order/limit → concat.
+ Promise.all(
+ chunk(authorIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select({ id: user.id, name: user.name, discriminator: user.discriminator })
+ .from(user)
+ .where(inArray(user.id, ids))
+ )
+ ).then((r) => r.flat()),
resolveReplyRefs(db, rows),
]);
const userById = new Map(users.map((u) => [u.id, u]));
@@ -700,11 +737,19 @@ export async function getInboxSnapshotForAgent(db: Database, botUserId: string):
const filtered = rows;
const senderIds = [...new Set(filtered.map((r) => r.latestSenderId).filter(Boolean))];
+ // Chunk the `inArray` for D1's 100-param limit — one distinct sender per
+ // pending channel, so >100 channels yields >100 ids; no order/limit → concat.
const users = senderIds.length
- ? await db
- .select({ id: user.id, name: user.name, discriminator: user.discriminator })
- .from(user)
- .where(inArray(user.id, senderIds))
+ ? (
+ await Promise.all(
+ chunk(senderIds, D1_MAX_IN_PARAMS).map((ids) =>
+ db
+ .select({ id: user.id, name: user.name, discriminator: user.discriminator })
+ .from(user)
+ .where(inArray(user.id, ids))
+ )
+ )
+ ).flat()
: [];
const userById = new Map(users.map((u) => [u.id, u]));
diff --git a/src/shared/src/db/queries/community/message.ts b/src/shared/src/db/queries/community/message.ts
index 96af064c5..6c7b92341 100644
--- a/src/shared/src/db/queries/community/message.ts
+++ b/src/shared/src/db/queries/community/message.ts
@@ -772,25 +772,33 @@ export async function getMessage(db: Database, messageId: string) {
// per-channel sequence without a separate lookup.
export async function getMessagesByIds(db: Database, ids: string[]) {
if (ids.length === 0) return [];
- const rows = await db
- .select({
- id: communityMessage.id,
- authorId: communityMessage.authorId,
- content: communityMessage.content,
- type: communityMessage.type,
- mentionType: communityMessage.mentionType,
- replyToId: communityMessage.replyToId,
- embeds: communityMessage.embeds,
- createdAt: communityMessage.createdAt,
- channelId: communityMessage.channelId,
- seq: communityMessage.seq,
- authorName: user.name,
- authorEmail: user.email,
- authorImage: user.image,
- })
- .from(communityMessage)
- .innerJoin(user, eq(communityMessage.authorId, user.id))
- .where(inArray(communityMessage.id, ids));
+ // `ids` are the thread/forum-post parents under one channel — unbounded on a
+ // busy forum. Chunk for D1's 100-param limit; no order/limit → concat.
+ const rows = (
+ await Promise.all(
+ chunk(ids, D1_MAX_IN_PARAMS).map((batch) =>
+ db
+ .select({
+ id: communityMessage.id,
+ authorId: communityMessage.authorId,
+ content: communityMessage.content,
+ type: communityMessage.type,
+ mentionType: communityMessage.mentionType,
+ replyToId: communityMessage.replyToId,
+ embeds: communityMessage.embeds,
+ createdAt: communityMessage.createdAt,
+ channelId: communityMessage.channelId,
+ seq: communityMessage.seq,
+ authorName: user.name,
+ authorEmail: user.email,
+ authorImage: user.image,
+ })
+ .from(communityMessage)
+ .innerJoin(user, eq(communityMessage.authorId, user.id))
+ .where(inArray(communityMessage.id, batch))
+ )
+ )
+ ).flat();
return rows.map((r) => ({ ...r, embeds: safeParseEmbeds(r.embeds, r.id) }));
}
@@ -885,25 +893,33 @@ export async function getMessageInScope(db: Database, messageId: string, scope:
/** Batched form of `getMessageInScope` — see its doc comment for the "why". */
export async function getMessagesByIdsInScope(db: Database, ids: string[], scope: MessageScope) {
if (ids.length === 0) return [];
- const rows = await db
- .select({
- id: communityMessage.id,
- authorId: communityMessage.authorId,
- content: communityMessage.content,
- type: communityMessage.type,
- mentionType: communityMessage.mentionType,
- replyToId: communityMessage.replyToId,
- embeds: communityMessage.embeds,
- seq: communityMessage.seq,
- createdAt: communityMessage.createdAt,
- channelId: communityMessage.channelId,
- authorName: user.name,
- discriminator: user.discriminator,
- authorEmail: user.email,
- authorImage: user.image,
- })
- .from(communityMessage)
- .innerJoin(user, eq(communityMessage.authorId, user.id))
- .where(and(inArray(communityMessage.id, ids), scopeCondition(scope)));
+ // `ids` are the reply-target ids of a message page (up to ~200), so this
+ // `inArray` is unbounded — chunk for D1's 100-param limit; no order/limit → concat.
+ const rows = (
+ await Promise.all(
+ chunk(ids, D1_MAX_IN_PARAMS).map((batch) =>
+ db
+ .select({
+ id: communityMessage.id,
+ authorId: communityMessage.authorId,
+ content: communityMessage.content,
+ type: communityMessage.type,
+ mentionType: communityMessage.mentionType,
+ replyToId: communityMessage.replyToId,
+ embeds: communityMessage.embeds,
+ seq: communityMessage.seq,
+ createdAt: communityMessage.createdAt,
+ channelId: communityMessage.channelId,
+ authorName: user.name,
+ discriminator: user.discriminator,
+ authorEmail: user.email,
+ authorImage: user.image,
+ })
+ .from(communityMessage)
+ .innerJoin(user, eq(communityMessage.authorId, user.id))
+ .where(and(inArray(communityMessage.id, batch), scopeCondition(scope)))
+ )
+ )
+ ).flat();
return rows.map((r) => ({ ...r, embeds: safeParseEmbeds(r.embeds, r.id) }));
}