A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {ChannelID, MessageID} from '@app/api/BrandedTypes';
import {BucketScanDirection, scanBucketsWithIndex} from '@app/api/channel/repositories/message/BucketScanEngine';
import {
advanceRow,
BatchBuilder,
deleteOneOrMany,
fetchMany,
fetchOne,
upsertOne,
} from '@app/api/database/CassandraQueryExecution';
import {Db, type QueryTemplate} from '@app/api/database/CassandraTypes';
import {buildPatchFromData, executeVersionedUpdate} from '@app/api/database/CassandraVersionedUpdate';
import type {ChannelMessageBucketRow, ChannelStateRow, MessageRow} from '@app/api/database/types/MessageTypes';
import {MESSAGE_COLUMNS} from '@app/api/database/types/MessageTypes';
import {Logger} from '@app/api/Logger';
import {Message} from '@app/api/models/Message';
import {
AttachmentLookup,
ChannelEmptyBuckets,
ChannelMessageBuckets,
ChannelPins,
ChannelState,
Messages,
MessagesByAuthorV2,
} from '@app/api/Tables';
import {startEarly} from '@app/api/utils/ConcurrencyUtils';
import {generateSnowflake} from '@fluxer/snowflake/src/Snowflake';
import * as BucketUtils from '@fluxer/snowflake/src/SnowflakeBuckets';
function getLogger() {
return Logger.child({module: 'MessageDataRepository'});
}
const DEFAULT_MESSAGE_LIMIT = 50;
const DEFAULT_BUCKET_INDEX_PAGE_SIZE = 200;
const LEGACY_BUCKETS_TO_CHECK = [0];
const FETCH_MESSAGE_BY_CHANNEL_BUCKET_AND_MESSAGE_ID = Messages.select({
where: [Messages.where.eq('channel_id'), Messages.where.eq('bucket'), Messages.where.eq('message_id')],
limit: 1,
});
const FETCH_CHANNEL_STATE = ChannelState.select({
where: ChannelState.where.eq('channel_id'),
limit: 1,
});
function memoizeQueryByLimit(build: (limit: number) => QueryTemplate): (limit: number) => QueryTemplate {
const cache = new Map<number, QueryTemplate>();
return (limit: number) => {
const cached = cache.get(limit);
if (cached) return cached;
const query = build(limit);
cache.set(limit, query);
return query;
};
}
const fetchMessagesBeforeQuery = memoizeQueryByLimit((limit) =>
Messages.select({
where: [
Messages.where.eq('channel_id'),
Messages.where.eq('bucket'),
Messages.where.lt('message_id', 'before_message_id'),
],
orderBy: {col: 'message_id', direction: 'DESC'},
limit,
}),
);
const fetchMessagesAfterDescQuery = memoizeQueryByLimit((limit) =>
Messages.select({
where: [
Messages.where.eq('channel_id'),
Messages.where.eq('bucket'),
Messages.where.gt('message_id', 'after_message_id'),
],
orderBy: {col: 'message_id', direction: 'DESC'},
limit,
}),
);
const fetchMessagesBetweenQuery = memoizeQueryByLimit((limit) =>
Messages.select({
where: [
Messages.where.eq('channel_id'),
Messages.where.eq('bucket'),
Messages.where.gt('message_id', 'after_message_id'),
Messages.where.lt('message_id', 'before_message_id'),
],
orderBy: {col: 'message_id', direction: 'DESC'},
limit,
}),
);
const fetchMessagesLatestDescQuery = memoizeQueryByLimit((limit) =>
Messages.select({
where: [Messages.where.eq('channel_id'), Messages.where.eq('bucket')],
orderBy: {col: 'message_id', direction: 'DESC'},
limit,
}),
);
const fetchMessagesAfterAscQuery = memoizeQueryByLimit((limit) =>
Messages.select({
where: [
Messages.where.eq('channel_id'),
Messages.where.eq('bucket'),
Messages.where.gt('message_id', 'after_message_id'),
],
orderBy: {col: 'message_id', direction: 'ASC'},
limit,
}),
);
const fetchMessagesOldestAscQuery = memoizeQueryByLimit((limit) =>
Messages.select({
where: [Messages.where.eq('channel_id'), Messages.where.eq('bucket')],
orderBy: {col: 'message_id', direction: 'ASC'},
limit,
}),
);
export class MessageDataRepository {
async listMessages(
channelId: ChannelID,
beforeMessageId?: MessageID,
limit: number = DEFAULT_MESSAGE_LIMIT,
afterMessageId?: MessageID,
): Promise<Array<Message>> {
if (limit <= 0) return [];
getLogger().debug(
{
channelId: channelId.toString(),
before: beforeMessageId?.toString() ?? null,
after: afterMessageId?.toString() ?? null,
limit,
},
'listMessages start',
);
if (beforeMessageId && afterMessageId) {
return this.listMessagesBetween(channelId, afterMessageId, beforeMessageId, limit);
}
if (beforeMessageId) {
return this.listMessagesBefore(channelId, beforeMessageId, limit);
}
if (afterMessageId) {
return this.listMessagesAfter(channelId, afterMessageId, limit);
}
return this.listMessagesLatest(channelId, limit);
}
private async listMessagesLatest(channelId: ChannelID, limit: number): Promise<Array<Message>> {
const state = await this.getChannelState(channelId);
const nowId = generateSnowflake();
const maxBucket = BucketUtils.makeBucket(nowId);
const minBucket = state?.created_bucket ?? BucketUtils.makeBucket(channelId);
return this.scanBucketsDescForMessages(channelId, {
limit,
minBucket,
maxBucket,
});
}
private async listMessagesBefore(channelId: ChannelID, before: MessageID, limit: number): Promise<Array<Message>> {
const state = await this.getChannelState(channelId);
const maxBucket = Math.min(BucketUtils.makeBucket(before), BucketUtils.makeBucket(generateSnowflake()));
const minBucket = state?.created_bucket ?? BucketUtils.makeBucket(channelId);
getLogger().debug(
{
channelId: channelId.toString(),
before: before.toString(),
limit,
maxBucket,
minBucket,
stateCreatedBucket: state?.created_bucket ?? null,
},
'listMessagesBefore: computed bucket range',
);
return this.scanBucketsDescForMessages(channelId, {
limit,
minBucket,
maxBucket,
before,
});
}
private async listMessagesAfter(channelId: ChannelID, after: MessageID, limit: number): Promise<Array<Message>> {
const state = await this.getChannelState(channelId);
const afterBucket = BucketUtils.makeBucket(after);
const createdMin = state?.created_bucket ?? BucketUtils.makeBucket(channelId);
const minBucket = Math.max(afterBucket, createdMin);
const nowBucket = BucketUtils.makeBucket(generateSnowflake());
const maxBucket = Math.max(nowBucket, minBucket);
getLogger().debug(
{
channelId: channelId.toString(),
action: 'listMessagesAfter',
after: after.toString(),
minBucket,
maxBucket,
limit,
},
'listMessagesAfter parameters',
);
const asc = await this.scanBucketsAscForMessages(channelId, {
limit,
minBucket,
maxBucket,
after,
});
return asc.reverse();
}
private async listMessagesBetween(
channelId: ChannelID,
after: MessageID,
before: MessageID,
limit: number,
): Promise<Array<Message>> {
const state = await this.getChannelState(channelId);
const afterBucket = BucketUtils.makeBucket(after);
const beforeBucket = BucketUtils.makeBucket(before);
const high = Math.max(afterBucket, beforeBucket);
const low = Math.min(afterBucket, beforeBucket);
const createdMin = state?.created_bucket ?? BucketUtils.makeBucket(channelId);
const minBucket = Math.max(low, createdMin);
const maxBucket = Math.min(high, BucketUtils.makeBucket(generateSnowflake()));
getLogger().debug(
{
channelId: channelId.toString(),
action: 'listMessagesBetween',
after: after.toString(),
before: before.toString(),
minBucket,
maxBucket,
limit,
},
'listMessagesBetween parameters',
);
return this.scanBucketsDescForMessages(channelId, {
limit,
minBucket,
maxBucket,
after,
before,
});
}
private async scanBucketsDescForMessages(
channelId: ChannelID,
opts: {
limit: number;
minBucket: number;
maxBucket: number;
before?: MessageID;
after?: MessageID;
},
): Promise<Array<Message>> {
const beforeBucket = opts.before ? BucketUtils.makeBucket(opts.before) : null;
const afterBucket = opts.after ? BucketUtils.makeBucket(opts.after) : null;
getLogger().debug(
{
channelId: channelId.toString(),
minBucket: opts.minBucket,
maxBucket: opts.maxBucket,
beforeBucket,
afterBucket,
},
'scanBucketsDescForMessages: starting scan',
);
const {rows: out} = await scanBucketsWithIndex<MessageRow>(
{
listBucketsFromIndex: async (query) =>
this.listBucketsDescFromIndex(channelId, {
minBucket: query.minBucket,
maxBucket: query.maxBucket,
limit: query.limit,
}),
fetchRowsForBucket: async (bucket, limit) =>
this.fetchRowsForBucket(channelId, bucket, limit, {
before: opts.before,
after: opts.after,
beforeBucket,
afterBucket,
}),
getRowId: (row) => row.message_id,
onEmptyUnboundedBucket: async (bucket) => this.markBucketEmpty(channelId, bucket),
onBucketHasRows: async (bucket) => this.touchBucketWithMessages(channelId, bucket),
},
{
limit: opts.limit,
minBucket: opts.minBucket,
maxBucket: opts.maxBucket,
direction: BucketScanDirection.Desc,
indexPageSize: DEFAULT_BUCKET_INDEX_PAGE_SIZE,
},
);
return this.finishScan(channelId, out);
}
private async scanBucketsAscForMessages(
channelId: ChannelID,
opts: {
limit: number;
minBucket: number;
maxBucket: number;
after: MessageID;
},
): Promise<Array<Message>> {
const afterBucket = BucketUtils.makeBucket(opts.after);
const {rows: out} = await scanBucketsWithIndex<MessageRow>(
{
listBucketsFromIndex: async (query) =>
this.listBucketsAscFromIndex(channelId, {
minBucket: query.minBucket,
maxBucket: query.maxBucket,
limit: query.limit,
}),
fetchRowsForBucket: async (bucket, limit) =>
this.fetchRowsForBucketAsc(channelId, bucket, limit, {
after: opts.after,
afterBucket,
}),
getRowId: (row) => row.message_id,
onEmptyUnboundedBucket: async (bucket) => this.markBucketEmpty(channelId, bucket),
onBucketHasRows: async (bucket) => this.touchBucketWithMessages(channelId, bucket),
},
{
limit: opts.limit,
minBucket: opts.minBucket,
maxBucket: opts.maxBucket,
direction: BucketScanDirection.Asc,
indexPageSize: DEFAULT_BUCKET_INDEX_PAGE_SIZE,
},
);
return this.finishScan(channelId, out);
}
private async finishScan(channelId: ChannelID, out: Array<MessageRow>): Promise<Array<Message>> {
if (out.length === 0) return [];
let maxId: MessageID = out[0].message_id;
let maxBucketForId = out[0].bucket;
for (const row of out) {
if (row.message_id > maxId) {
maxId = row.message_id;
maxBucketForId = row.bucket;
}
}
await this.touchChannelHasMessages(channelId);
await this.advanceChannelStateLastMessageIfNewer(channelId, maxId, maxBucketForId);
return this.repairAndMapMessages(channelId, out);
}
private async fetchRowsForBucketAsc(
channelId: ChannelID,
bucket: number,
limit: number,
meta: {
after: MessageID;
afterBucket: number;
},
): Promise<{
rows: Array<MessageRow>;
unbounded: boolean;
}> {
getLogger().debug(
{
channelId: channelId.toString(),
bucket,
limit,
meta: {after: meta.after.toString(), afterBucket: meta.afterBucket},
},
'fetchRowsForBucketAsc parameters',
);
if (bucket === meta.afterBucket) {
const q = fetchMessagesAfterAscQuery(limit);
const rows = await fetchMany<MessageRow>(
q.bind({
channel_id: channelId,
bucket,
after_message_id: meta.after,
}),
);
return {rows, unbounded: false};
}
const q = fetchMessagesOldestAscQuery(limit);
const rows = await fetchMany<MessageRow>(q.bind({channel_id: channelId, bucket}));
return {rows, unbounded: true};
}
private async fetchRowsForBucket(
channelId: ChannelID,
bucket: number,
limit: number,
meta: {
before?: MessageID;
after?: MessageID;
beforeBucket: number | null;
afterBucket: number | null;
},
): Promise<{
rows: Array<MessageRow>;
unbounded: boolean;
}> {
getLogger().debug(
{
channelId: channelId.toString(),
bucket,
limit,
meta: {
before: meta.before?.toString() ?? null,
after: meta.after?.toString() ?? null,
beforeBucket: meta.beforeBucket,
afterBucket: meta.afterBucket,
},
},
'fetchRowsForBucket parameters',
);
if (meta.before && meta.after && meta.beforeBucket === bucket && meta.afterBucket === bucket) {
const q = fetchMessagesBetweenQuery(limit);
const rows = await fetchMany<MessageRow>(
q.bind({
channel_id: channelId,
bucket,
after_message_id: meta.after,
before_message_id: meta.before,
}),
);
return {rows, unbounded: false};
}
if (meta.before && meta.beforeBucket === bucket) {
const q = fetchMessagesBeforeQuery(limit);
const rows = await fetchMany<MessageRow>(
q.bind({
channel_id: channelId,
bucket,
before_message_id: meta.before,
}),
);
return {rows, unbounded: false};
}
if (meta.after && meta.afterBucket === bucket) {
const q = fetchMessagesAfterDescQuery(limit);
const rows = await fetchMany<MessageRow>(
q.bind({
channel_id: channelId,
bucket,
after_message_id: meta.after,
}),
);
return {rows, unbounded: false};
}
const q = fetchMessagesLatestDescQuery(limit);
const rows = await fetchMany<MessageRow>(q.bind({channel_id: channelId, bucket}));
return {rows, unbounded: true};
}
private async touchBucketWithMessages(channelId: ChannelID, bucket: number): Promise<void> {
const batch = new BatchBuilder();
batch.addPrepared(
ChannelMessageBuckets.upsertAll({
channel_id: channelId,
bucket,
updated_at: new Date(),
}),
);
batch.addPrepared(
ChannelEmptyBuckets.deleteByPk({
channel_id: channelId,
bucket,
}),
);
await batch.execute(false);
}
private async markBucketEmpty(channelId: ChannelID, bucket: number): Promise<void> {
const batch = new BatchBuilder();
batch.addPrepared(
ChannelMessageBuckets.deleteByPk({
channel_id: channelId,
bucket,
}),
);
batch.addPrepared(
ChannelEmptyBuckets.upsertAll({
channel_id: channelId,
bucket,
updated_at: new Date(),
}),
);
await batch.execute(false);
}
private async touchChannelHasMessages(channelId: ChannelID): Promise<void> {
await upsertOne(
ChannelState.patchByPk(
{channel_id: channelId},
{
has_messages: Db.set(true),
updated_at: Db.set(new Date()),
},
),
);
}
private async advanceChannelStateLastMessageIfNewer(
channelId: ChannelID,
newLastMessageId: MessageID,
newLastMessageBucket: number,
): Promise<void> {
const set = {
has_messages: true,
last_message_id: newLastMessageId,
last_message_bucket: newLastMessageBucket,
updated_at: new Date(),
};
await advanceRow(
{
table: ChannelState,
pk: {channel_id: channelId},
guard: {col: 'last_message_id', value: newLastMessageId},
set,
insert: {channel_id: channelId, ...set},
},
async () => {
const state = await this.getChannelState(channelId);
const prev = state?.last_message_id ?? null;
if (prev !== null && newLastMessageId <= prev) return false;
await upsertOne(
ChannelState.patchByPk(
{channel_id: channelId},
{
has_messages: Db.set(set.has_messages),
last_message_id: Db.set(set.last_message_id),
last_message_bucket: Db.set(set.last_message_bucket),
updated_at: Db.set(set.updated_at),
},
),
);
return true;
},
);
}
private async recordChannelStateForMessage(
channelId: ChannelID,
messageId: MessageID,
messageBucket: number,
): Promise<void> {
const always = {
created_bucket: BucketUtils.makeBucket(channelId),
has_messages: true,
updated_at: new Date(),
};
const set = {last_message_id: messageId, last_message_bucket: messageBucket};
await advanceRow(
{
table: ChannelState,
pk: {channel_id: channelId},
guard: {col: 'last_message_id', value: messageId},
set,
always,
insert: {channel_id: channelId, ...always, ...set},
},
async () => {
const state = await this.getChannelState(channelId);
const prev = state?.last_message_id ?? null;
const patch = {
created_bucket: Db.set(always.created_bucket),
has_messages: Db.set(always.has_messages),
updated_at: Db.set(always.updated_at),
};
await upsertOne(
ChannelState.patchByPk(
{channel_id: channelId},
prev !== null && messageId <= prev
? patch
: {...patch, last_message_id: Db.set(messageId), last_message_bucket: Db.set(messageBucket)},
),
);
return true;
},
);
}
private async getChannelState(channelId: ChannelID): Promise<ChannelStateRow | null> {
return fetchOne<ChannelStateRow>(FETCH_CHANNEL_STATE.bind({channel_id: channelId}));
}
private async listBucketsDescFromIndex(
channelId: ChannelID,
opts: {
minBucket?: number;
maxBucket?: number;
limit: number;
},
): Promise<Array<number>> {
const where = [ChannelMessageBuckets.where.eq('channel_id')];
if (typeof opts.minBucket === 'number') where.push(ChannelMessageBuckets.where.gte('bucket', 'min_bucket'));
if (typeof opts.maxBucket === 'number') where.push(ChannelMessageBuckets.where.lte('bucket', 'max_bucket'));
const q = ChannelMessageBuckets.select({
columns: ['bucket'],
where,
orderBy: {col: 'bucket', direction: 'DESC'},
limit: opts.limit,
});
const params = {
channel_id: channelId,
...(typeof opts.minBucket === 'number' ? {min_bucket: opts.minBucket} : {}),
...(typeof opts.maxBucket === 'number' ? {max_bucket: opts.maxBucket} : {}),
};
const rows = await fetchMany<Pick<ChannelMessageBucketRow, 'bucket'>>(q.bind(params));
const buckets = rows.map((r) => r.bucket);
getLogger().debug(
{
channelId: channelId.toString(),
minBucket: opts.minBucket ?? null,
maxBucket: opts.maxBucket ?? null,
limit: opts.limit,
bucketsFound: buckets,
},
'listBucketsDescFromIndex: query result',
);
return buckets;
}
private async listBucketsAscFromIndex(
channelId: ChannelID,
opts: {
minBucket?: number;
maxBucket?: number;
limit: number;
},
): Promise<Array<number>> {
const where = [ChannelMessageBuckets.where.eq('channel_id')];
if (typeof opts.minBucket === 'number') where.push(ChannelMessageBuckets.where.gte('bucket', 'min_bucket'));
if (typeof opts.maxBucket === 'number') where.push(ChannelMessageBuckets.where.lte('bucket', 'max_bucket'));
const q = ChannelMessageBuckets.select({
columns: ['bucket'],
where,
orderBy: {col: 'bucket', direction: 'ASC'},
limit: opts.limit,
});
const params = {
channel_id: channelId,
...(typeof opts.minBucket === 'number' ? {min_bucket: opts.minBucket} : {}),
...(typeof opts.maxBucket === 'number' ? {max_bucket: opts.maxBucket} : {}),
};
const rows = await fetchMany<Pick<ChannelMessageBucketRow, 'bucket'>>(q.bind(params));
return rows.map((r) => r.bucket);
}
async getMessage(channelId: ChannelID, messageId: MessageID): Promise<Message | null> {
const bucket = BucketUtils.makeBucket(messageId);
const message = await fetchOne<MessageRow>(
FETCH_MESSAGE_BY_CHANNEL_BUCKET_AND_MESSAGE_ID.bind({
channel_id: channelId,
bucket,
message_id: messageId,
}),
);
if (message) {
if (this.isCorruptMessageRow(message)) {
void this.purgeCorruptMessageRow(channelId, message);
return null;
}
return new Message(message);
}
const repairedMessage = await this.attemptBucketReadRepair(channelId, messageId, bucket);
return repairedMessage;
}
async upsertMessage(
data: MessageRow,
oldData?: MessageRow | null,
alongsideChannelState?: () => Promise<void>,
): Promise<Message> {
const expectedBucket = BucketUtils.makeBucket(data.message_id);
if (data.bucket !== expectedBucket) {
throw new Error(
`Invalid message bucket for ${data.message_id.toString()}: expected ${expectedBucket}, received ${data.bucket}`,
);
}
const batch = new BatchBuilder();
batch.addPrepared(
ChannelEmptyBuckets.deleteByPk({
channel_id: data.channel_id,
bucket: data.bucket,
}),
);
const result = await executeVersionedUpdate<MessageRow, 'channel_id' | 'bucket' | 'message_id'>(
async () => {
const pk = {
channel_id: data.channel_id,
bucket: data.bucket,
message_id: data.message_id,
};
const existingMessage = await fetchOne<MessageRow>(FETCH_MESSAGE_BY_CHANNEL_BUCKET_AND_MESSAGE_ID.bind(pk));
return existingMessage ?? null;
},
(current) => ({
pk: {
channel_id: data.channel_id,
bucket: data.bucket,
message_id: data.message_id,
},
patch: buildPatchFromData(data, current, MESSAGE_COLUMNS, ['channel_id', 'bucket', 'message_id']),
}),
Messages,
{initialData: oldData},
);
const finalVersion = result.finalVersion ?? 1;
if (data.author_id != null) {
batch.addPrepared(
MessagesByAuthorV2.upsertAll({
author_id: data.author_id,
channel_id: data.channel_id,
message_id: data.message_id,
}),
);
}
if (data.pinned_timestamp) {
batch.addPrepared(
ChannelPins.upsertAll({
channel_id: data.channel_id,
message_id: data.message_id,
pinned_timestamp: data.pinned_timestamp,
}),
);
}
if (oldData?.pinned_timestamp && !data.pinned_timestamp) {
batch.addPrepared(
ChannelPins.deleteByPk({
channel_id: data.channel_id,
message_id: data.message_id,
pinned_timestamp: oldData.pinned_timestamp,
}),
);
}
if (oldData?.attachments) {
for (const attachment of oldData.attachments) {
batch.addPrepared(
AttachmentLookup.deleteByPk({
channel_id: data.channel_id,
attachment_id: attachment.attachment_id,
filename: attachment.filename,
}),
);
}
}
if (data.attachments) {
for (const attachment of data.attachments) {
batch.addPrepared(
AttachmentLookup.upsertAll({
channel_id: data.channel_id,
attachment_id: attachment.attachment_id,
filename: attachment.filename,
message_id: data.message_id,
}),
);
}
}
batch.addPrepared(
ChannelMessageBuckets.upsertAll({
channel_id: data.channel_id,
bucket: data.bucket,
updated_at: new Date(),
}),
);
await batch.execute(false);
const channelStateTask = startEarly(
this.recordChannelStateForMessage(data.channel_id, data.message_id, data.bucket),
);
const alongsideTask = startEarly(alongsideChannelState ? alongsideChannelState() : Promise.resolve());
await channelStateTask;
await alongsideTask;
return new Message({...data, version: finalVersion});
}
async updateEmbeds(message: Message): Promise<void> {
await upsertOne(
Messages.patchByPk(
{
channel_id: message.channelId,
bucket: message.bucket,
message_id: message.id,
},
{
embeds: Db.set(message.embeds.length > 0 ? message.embeds.map((e) => e.toMessageEmbed()) : null),
},
),
);
}
private async attemptBucketReadRepair(
channelId: ChannelID,
messageId: MessageID,
expectedBucket: number,
): Promise<Message | null> {
for (const legacyBucket of LEGACY_BUCKETS_TO_CHECK) {
if (legacyBucket === expectedBucket) continue;
const legacyRow = await fetchOne<MessageRow>(
FETCH_MESSAGE_BY_CHANNEL_BUCKET_AND_MESSAGE_ID.bind({
channel_id: channelId,
bucket: legacyBucket,
message_id: messageId,
}),
);
if (!legacyRow) continue;
Logger.warn(
{channelId: channelId.toString(), messageId: messageId.toString(), legacyBucket, expectedBucket},
'Repairing message bucket mismatch',
);
const repairedRow: MessageRow = {
...legacyRow,
bucket: expectedBucket,
};
const repairedMessage = await this.upsertMessage(repairedRow, legacyRow);
await deleteOneOrMany(
Messages.deleteByPk({
channel_id: channelId,
bucket: legacyBucket,
message_id: messageId,
}),
);
return repairedMessage;
}
return null;
}
private async repairAndMapMessages(channelId: ChannelID, messages: Array<MessageRow>): Promise<Array<Message>> {
if (messages.length === 0) return [];
const repaired: Array<Message> = [];
for (const message of messages) {
if (this.isCorruptMessageRow(message)) {
void this.purgeCorruptMessageRow(channelId, message);
continue;
}
const expectedBucket = BucketUtils.makeBucket(message.message_id);
if (message.bucket === expectedBucket) {
repaired.push(new Message(message));
continue;
}
const repairedMessage = await this.attemptBucketReadRepair(channelId, message.message_id, expectedBucket);
if (repairedMessage) {
repaired.push(repairedMessage);
continue;
}
Logger.warn(
{
channelId: channelId.toString(),
messageId: message.message_id.toString(),
legacyBucket: message.bucket,
expectedBucket,
},
'Failed to repair message bucket mismatch during listMessages; returning legacy row',
);
repaired.push(new Message(message));
}
return repaired;
}
private isCorruptMessageRow(row: MessageRow): boolean {
const hasAuthor = row.author_id !== null && row.author_id !== undefined;
const hasWebhook = row.webhook_id !== null && row.webhook_id !== undefined;
return !hasAuthor && !hasWebhook;
}
private async purgeCorruptMessageRow(channelId: ChannelID, row: MessageRow): Promise<void> {
try {
await deleteOneOrMany(
Messages.deleteByPk({
channel_id: channelId,
bucket: row.bucket,
message_id: row.message_id,
}),
);
getLogger().warn(
{
channelId: channelId.toString(),
messageId: row.message_id.toString(),
bucket: row.bucket,
},
'Read-repair: deleted partial message row left by a concurrent edit/delete race',
);
} catch (error) {
getLogger().debug(
{error, channelId: channelId.toString(), messageId: row.message_id.toString()},
'Read-repair purge of partial message row failed; will retry on next read',
);
}
}
}