Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
8.5 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import {type ChannelID, createChannelID, createMessageID, type MessageID, type UserID} from '@app/api/BrandedTypes';
import type {MessageRepository} from '@app/api/channel/repositories/MessageRepository';
import type {ThreadViewer} from '@app/api/experiment/ChannelThreadsGate';
import {Logger} from '@app/api/Logger';
import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware';
import type {Message} from '@app/api/models/Message';
import {buildMessageSearchFilters} from '@app/api/search/BuildMessageSearchFilters';
import type {IMessageSearchService} from '@app/api/search/IMessageSearchService';
import {deleteMessageSearchDocuments} from '@app/api/search/MessageSearchIndexCleanup';
import type {MessageSearchResponseMapper} from '@app/api/search/MessageSearchResponseMapper';
import {mapWithConcurrency} from '@app/api/utils/ConcurrencyUtils';
import type {SearchResult} from '@fluxer/schema/src/contracts/search/SearchAdapterTypes';
import type {MessageSearchFilters, SearchableMessage} from '@fluxer/schema/src/contracts/search/SearchDocumentTypes';
import type {MessageSearchRequest} from '@fluxer/schema/src/domains/message/MessageRequestSchemas';
import type {MessageSearchResultsResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas';

const RECONCILE_BATCH_SIZE = 250;
const MAX_RECONCILE_PAGES = 40;
const MAX_STALE_DELETE_ABSOLUTE = 250;
const MAX_STALE_DELETE_RATIO = 0.5;
const HIT_LOOKUP_CONCURRENCY = 32;

interface MessageLookupRepository {
	readonly messages: Pick<MessageRepository, 'getMessage'>;
}

interface MessageSearchLookupResult extends SearchResult<SearchableMessage> {
	messages: Array<Message>;
}

interface SearchExistingMessagesParams {
	searchService: IMessageSearchService;
	messageRepository: MessageLookupRepository;
	query: string;
	filters: MessageSearchFilters;
	hitsPerPage: number;
	page: number;
	cursor?: Array<string>;
}

interface ValidatedHit {
	hit: SearchableMessage;
	message: Message | null;
}

interface ValidatedHits {
	validHits: Array<ValidatedHit>;
	staleMessageIds: Array<MessageID>;
	lookupErrorCount: number;
}

function resolvedMessages(validHits: Array<ValidatedHit>): Array<Message> {
	return validHits.map((entry) => entry.message).filter((message): message is Message => message !== null);
}

export async function searchExistingMessages({
	searchService,
	messageRepository,
	query,
	filters,
	hitsPerPage,
	page,
	cursor,
}: SearchExistingMessagesParams): Promise<MessageSearchLookupResult> {
	const result = await searchService.searchMessages(query, filters, {
		hitsPerPage,
		page: cursor?.length ? undefined : page,
		cursor,
	});
	const validated = await validateSearchHits(messageRepository, result.hits);
	if (validated.staleMessageIds.length === 0) {
		return {...result, messages: resolvedMessages(validated.validHits)};
	}
	if (cursor?.length) {
		if (validated.lookupErrorCount === 0) {
			await deleteStaleSearchDocuments(searchService, validated.staleMessageIds, result.hits.length);
		}
		return {
			...result,
			hits: validated.validHits.map((entry) => entry.hit),
			total: Math.max(validated.validHits.length, result.total - validated.staleMessageIds.length),
			messages: resolvedMessages(validated.validHits),
		};
	}
	return reconcileOffsetSearchResult({
		searchService,
		messageRepository,
		query,
		filters,
		hitsPerPage,
		page,
	});
}

export function emptyMessageSearchResponse(searchParams: MessageSearchRequest): MessageSearchResultsResponse {
	return {
		channels: [],
		messages: [],
		total: 0,
		hits_per_page: searchParams.hits_per_page ?? 25,
		page: searchParams.page ?? 1,
	};
}

export async function runMessageSearch({
	searchService,
	messageRepository,
	responseMapper,
	filterParams,
	searchParams,
	channelIds,
	userId,
	viewer,
	requestCache,
}: {
	searchService: IMessageSearchService;
	messageRepository: MessageLookupRepository;
	responseMapper: MessageSearchResponseMapper;
	filterParams: MessageSearchRequest;
	searchParams: MessageSearchRequest;
	channelIds: Array<string>;
	userId: UserID;
	viewer: ThreadViewer;
	requestCache: RequestCache;
}): Promise<MessageSearchResultsResponse> {
	const hitsPerPage = searchParams.hits_per_page ?? 25;
	const page = searchParams.page ?? 1;
	const result = await searchExistingMessages({
		searchService,
		messageRepository,
		query: searchParams.content ?? '',
		filters: buildMessageSearchFilters(filterParams, channelIds),
		hitsPerPage,
		page,
		cursor: searchParams.cursor,
	});
	const mappedResponses = await responseMapper.mapSearchResultToResponses(
		result.messages,
		userId,
		viewer,
		requestCache,
	);
	return {
		messages: mappedResponses.messages,
		channels: mappedResponses.channels,
		total: result.total,
		hits_per_page: hitsPerPage,
		page,
		cursor: result.cursor,
		...(mappedResponses.threads ? {threads: mappedResponses.threads, members: mappedResponses.members} : {}),
	};
}

async function reconcileOffsetSearchResult({
	searchService,
	messageRepository,
	query,
	filters,
	hitsPerPage,
	page,
}: Omit<SearchExistingMessagesParams, 'cursor'>): Promise<MessageSearchLookupResult> {
	const requestedOffset = (page - 1) * hitsPerPage;
	const pageHits: Array<ValidatedHit> = [];
	const staleMessageIds: Array<MessageID> = [];
	let lookupErrorCount = 0;
	let examinedCount = 0;
	let validTotal = 0;
	let rawOffset = 0;
	let rawPage = 1;
	let corpusTotal = 0;
	while (rawPage <= MAX_RECONCILE_PAGES) {
		const result = await searchService.searchMessages(query, filters, {
			hitsPerPage: RECONCILE_BATCH_SIZE,
			page: rawPage,
		});
		corpusTotal = result.total;
		if (result.hits.length === 0) {
			break;
		}
		const validated = await validateSearchHits(messageRepository, result.hits);
		lookupErrorCount += validated.lookupErrorCount;
		examinedCount += result.hits.length;
		staleMessageIds.push(...validated.staleMessageIds);
		for (const entry of validated.validHits) {
			if (validTotal >= requestedOffset && pageHits.length < hitsPerPage) {
				pageHits.push(entry);
			}
			validTotal += 1;
		}
		rawOffset += result.hits.length;
		if (pageHits.length >= hitsPerPage && rawOffset >= requestedOffset) {
			break;
		}
		if (rawOffset >= result.total) {
			break;
		}
		rawPage += 1;
	}
	if (lookupErrorCount === 0) {
		await deleteStaleSearchDocuments(searchService, staleMessageIds, examinedCount);
	}
	return {
		hits: pageHits.map((entry) => entry.hit),
		total: Math.max(pageHits.length, corpusTotal - staleMessageIds.length),
		messages: resolvedMessages(pageHits),
	};
}

async function validateSearchHits(
	messageRepository: MessageLookupRepository,
	hits: Array<SearchableMessage>,
): Promise<ValidatedHits> {
	const checked = await mapWithConcurrency(hits, HIT_LOOKUP_CONCURRENCY, async (hit) => {
		let channelId: ChannelID;
		let messageId: MessageID;
		try {
			channelId = createChannelID(BigInt(hit.channelId));
			messageId = createMessageID(BigInt(hit.id));
		} catch (_invalidId) {
			return {entry: null, staleMessageId: null, lookupError: false};
		}
		let message: Message | null;
		try {
			message = await messageRepository.messages.getMessage(channelId, messageId);
		} catch (error) {
			Logger.warn(
				{error, messageId: hit.id, channelId: hit.channelId},
				'Search read repair lookup failed; keeping document',
			);
			return {entry: {hit, message: null}, staleMessageId: null, lookupError: true};
		}
		if (message && message.channelId.toString() === hit.channelId) {
			return {entry: {hit, message}, staleMessageId: null, lookupError: false};
		}
		return {entry: null, staleMessageId: messageId, lookupError: false};
	});
	const validHits: Array<ValidatedHit> = [];
	const staleMessageIds: Array<MessageID> = [];
	let lookupErrorCount = 0;
	for (const item of checked) {
		if (item.entry) {
			validHits.push(item.entry);
		}
		if (item.staleMessageId) {
			staleMessageIds.push(item.staleMessageId);
		}
		if (item.lookupError) {
			lookupErrorCount += 1;
		}
	}
	return {validHits, staleMessageIds, lookupErrorCount};
}

async function deleteStaleSearchDocuments(
	searchService: IMessageSearchService,
	messageIds: Array<MessageID>,
	examinedCount: number,
): Promise<void> {
	if (messageIds.length === 0) {
		return;
	}
	if (
		messageIds.length > MAX_STALE_DELETE_ABSOLUTE ||
		(examinedCount > 0 && messageIds.length / examinedCount > MAX_STALE_DELETE_RATIO)
	) {
		Logger.warn(
			{staleMessageCount: messageIds.length, examinedCount},
			'Search read repair delete exceeded safety cap; skipping delete',
		);
		return;
	}
	await deleteMessageSearchDocuments(messageIds, {
		searchService,
		context: {source: 'message_search_read_repair', staleMessageCount: messageIds.length},
	});
}