A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {ChannelID, MessageID, UserID} from '@app/api/BrandedTypes';
import {createChannelID} from '@app/api/BrandedTypes';
import {isMessageSearchIndexable} from '@app/api/channel/services/message/MessageSearchService';
import {Logger} from '@app/api/Logger';
import {getMessageSearchService} from '@app/api/SearchFactory';
import type {IMessageSearchService} from '@app/api/search/IMessageSearchService';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {seconds} from 'itty-time';
import {z} from 'zod';
const PayloadSchema = z.object({
channelId: z.string(),
completionKey: z.string().optional(),
channelCount: z.number().optional(),
});
const BULK_BATCH_SIZE = 5000;
const COMPLETION_KEY_TTL_SECONDS = seconds('1 day');
export async function indexChannelMessageHistory(
searchService: IMessageSearchService,
channelId: ChannelID,
): Promise<number> {
const {channelRepository, userRepository} = getWorkerDependencies();
const authorBotCache = new Map<UserID, boolean>();
let cursor: MessageID | undefined;
let totalIndexed = 0;
while (true) {
const messages = await channelRepository.messages.listMessages(channelId, cursor, BULK_BATCH_SIZE);
if (messages.length === 0) {
break;
}
const newAuthorIds = [
...new Set(messages.map((m) => m.authorId).filter((id): id is UserID => id !== null && !authorBotCache.has(id))),
];
if (newAuthorIds.length > 0) {
const users = await userRepository.listUsers(newAuthorIds);
for (const user of users) {
authorBotCache.set(user.id, user.isBot);
}
}
const indexable = messages.filter(isMessageSearchIndexable);
await searchService.bulkIndexMessages(indexable, authorBotCache);
totalIndexed += messages.length;
cursor = messages[messages.length - 1]!.id;
Logger.debug(
{channelId: channelId.toString(), indexed: totalIndexed, batch: messages.length},
'Bulk indexed message batch',
);
if (messages.length < BULK_BATCH_SIZE) {
break;
}
}
const channel = await channelRepository.channelData.findUnique(channelId);
if (channel?.isThread()) {
await channelRepository.channelData.patchIndexedAt(channelId, new Date());
} else if (channel) {
await channelRepository.channelData.upsert({...channel.toRow(), indexed_at: new Date()}, channel.toRow());
}
return totalIndexed;
}
const indexChannelMessages: WorkerTaskHandler = async (payload) => {
const validated = PayloadSchema.parse(payload);
const searchService = getMessageSearchService();
if (!searchService) {
throw new Error('Search service is not available in this worker process');
}
const channelId = createChannelID(BigInt(validated.channelId));
const {kvClient} = getWorkerDependencies();
try {
const totalIndexed = await indexChannelMessageHistory(searchService, channelId);
if (validated.completionKey && validated.channelCount) {
await kvClient.sadd(validated.completionKey, validated.channelId);
await kvClient.expire(validated.completionKey, COMPLETION_KEY_TTL_SECONDS);
const completed = await kvClient.scard(validated.completionKey);
if (completed >= validated.channelCount) {
try {
await searchService.refreshIndex();
} catch (error) {
Logger.warn({error}, 'Search refresh after bulk indexing failed');
}
await kvClient.del(validated.completionKey);
Logger.info({completionKey: validated.completionKey}, 'All channels indexed');
}
}
Logger.info({channelId: channelId.toString(), totalIndexed}, 'Bulk channel indexing complete');
} catch (error) {
Logger.error({error, channelId: channelId.toString()}, 'Failed to bulk index channel messages');
throw error;
}
};
export default indexChannelMessages;