A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
import {stripOwnAttachmentSignature} from '@app/api/attachment/AttachmentUrls';
import type {ChannelID, MessageID} from '@app/api/BrandedTypes';
import type {RichEmbedMediaWithMetadata} from '@app/api/channel/EmbedTypes';
import type {MessageEmbed, MessageEmbedChild} from '@app/api/database/types/MessageTypes';
import {
type IMediaService,
type MediaProxyMetadataResponse,
type MediaProxyNsfwMode,
mediaProxyMetadataPolicy,
} from '@app/api/infrastructure/IMediaService';
import type {IUnfurlerService, UnfurlOptions} from '@app/api/infrastructure/IUnfurlerService';
import {Logger} from '@app/api/Logger';
import {Embed} from '@app/api/models/Embed';
import {EmbedAuthor} from '@app/api/models/EmbedAuthor';
import {EmbedField} from '@app/api/models/EmbedField';
import {EmbedFooter} from '@app/api/models/EmbedFooter';
import {EmbedMedia} from '@app/api/models/EmbedMedia';
import * as UnfurlerUtils from '@app/api/utils/UnfurlerUtils';
import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError';
import {EmbedMediaFlags} from '@fluxer/constants/src/ChannelConstants';
import {MAX_EMBEDS_PER_MESSAGE} from '@fluxer/constants/src/LimitConstants';
import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError';
import type {MessageEmbedChildResponse, MessageEmbedResponse} from '@fluxer/schema/src/domains/message/EmbedSchemas';
import type {
RichEmbedAuthorRequest,
RichEmbedFooterRequest,
RichEmbedMediaRequest,
RichEmbedRequest,
} from '@fluxer/schema/src/domains/message/MessageRequestSchemas';
import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
interface ProcessedUrlEmbeds {
embeds: Array<Embed>;
}
interface InitialUrlEmbedResult {
embeds: Array<MessageEmbed>;
hasUncachedUrls: boolean;
}
type RichEmbedRequestWithMedia = RichEmbedRequest & {
image?: RichEmbedMediaWithMetadata | null;
thumbnail?: RichEmbedMediaWithMetadata | null;
};
function canonicalUrl(url: string | null | undefined): string | null {
return url == null ? null : stripOwnAttachmentSignature(url);
}
export class EmbedService {
private readonly MAX_EMBED_CHARACTERS = 6000;
private readonly MAX_EMBED_CHARACTERS_BUG_HUNTER = 12000;
constructor(
private unfurlerService: IUnfurlerService,
private mediaService: IMediaService,
private workerService: IWorkerService<WorkerTaskName>,
) {}
async getInitialEmbeds(params: {
content: string | null;
customEmbeds?: Array<RichEmbedRequest>;
nsfwMode: MediaProxyNsfwMode;
isBugHunterBot?: boolean;
}): Promise<{
embeds: Array<MessageEmbed> | null;
hasUncachedUrls: boolean;
}> {
if (params.customEmbeds?.length) {
this.validateEmbedSize(params.customEmbeds, params.isBugHunterBot ?? false);
const embeds = await Promise.all(params.customEmbeds.map((embed) => this.createEmbed(embed, params.nsfwMode)));
return {embeds: embeds.map((embed) => embed.toMessageEmbed()), hasUncachedUrls: false};
}
if (!params.content) {
return {embeds: null, hasUncachedUrls: false};
}
const urls = UnfurlerUtils.extractURLs(params.content);
if (!urls.length) {
return {embeds: null, hasUncachedUrls: false};
}
return this.getInitialUrlEmbeds(urls, params.nsfwMode);
}
async enqueueUrlEmbedExtraction(
channelId: ChannelID,
messageId: MessageID,
guildId: bigint | null,
nsfwMode: MediaProxyNsfwMode,
options: {content?: string | null} = {},
): Promise<void> {
await this.enqueue(channelId, messageId, guildId, nsfwMode, options);
}
async processUrlWithCachePolicy(
url: string,
nsfwMode: MediaProxyNsfwMode = 'block',
options: UnfurlOptions = {},
): Promise<ProcessedUrlEmbeds> {
const result = await this.unfurlerService.unfurlWithCachePolicy(url, nsfwMode, options);
return {
embeds: result.embeds.map((embedData) => new Embed(this.mapResponseEmbed(embedData))),
};
}
private async getInitialUrlEmbeds(
urls: Array<string>,
nsfwMode: MediaProxyNsfwMode,
): Promise<{
embeds: Array<MessageEmbed> | null;
hasUncachedUrls: boolean;
}> {
const resolved = await Promise.all(urls.map((url) => this.getCachedUrlEmbeds(url, nsfwMode)));
const orderedEmbeds: Array<MessageEmbed> = [];
let hasUncachedUrls = false;
for (const result of resolved) {
if (result.hasUncachedUrls) {
hasUncachedUrls = true;
}
orderedEmbeds.push(...result.embeds);
if (orderedEmbeds.length >= MAX_EMBEDS_PER_MESSAGE) {
return {
embeds: orderedEmbeds.slice(0, MAX_EMBEDS_PER_MESSAGE),
hasUncachedUrls,
};
}
}
return {
embeds: orderedEmbeds.length > 0 ? orderedEmbeds : null,
hasUncachedUrls,
};
}
private async getCachedUrlEmbeds(url: string, nsfwMode: MediaProxyNsfwMode): Promise<InitialUrlEmbedResult> {
try {
const {embeds} = await this.processUrlWithCachePolicy(url, nsfwMode, {cacheOnly: true});
if (embeds.length === 0) {
return {embeds: [], hasUncachedUrls: true};
}
const messageEmbeds = embeds.map((embed) => embed.toMessageEmbed());
const safeEmbeds =
nsfwMode === 'allow' ? messageEmbeds : messageEmbeds.filter((embed) => !this.isEmbedNsfw(embed));
return {embeds: safeEmbeds, hasUncachedUrls: false};
} catch (error) {
Logger.warn({error, url}, 'Failed to read cached URL embeds during message creation');
return {embeds: [], hasUncachedUrls: true};
}
}
private isEmbedNsfw(embed: MessageEmbed): boolean {
if (embed.nsfw) return true;
for (const child of embed.children ?? []) {
if (child.nsfw) return true;
}
return false;
}
private mapResponseEmbed(embed: MessageEmbedResponse): MessageEmbed {
return {
...this.mapResponseEmbedChild(embed),
children:
embed.children && embed.children.length > 0
? embed.children.map((child) => this.mapResponseEmbedChild(child))
: null,
};
}
private mapResponseEmbedChild(embed: MessageEmbedChildResponse): MessageEmbedChild {
return {
type: embed.type ?? null,
title: embed.title ?? null,
description: embed.description ?? null,
url: canonicalUrl(embed.url),
timestamp: embed.timestamp ? new Date(embed.timestamp) : null,
color: embed.color ?? null,
author: embed.author
? {
name: embed.author.name ?? null,
url: canonicalUrl(embed.author.url),
icon_url: canonicalUrl(embed.author.icon_url),
}
: null,
provider: embed.provider
? {
name: embed.provider.name ?? null,
url: canonicalUrl(embed.provider.url),
}
: null,
thumbnail: this.mapResponseMedia(embed.thumbnail),
image: this.mapResponseMedia(embed.image),
video: this.mapResponseMedia(embed.video),
audio: this.mapResponseMedia(embed.audio),
footer: embed.footer
? {
text: embed.footer.text ?? null,
icon_url: canonicalUrl(embed.footer.icon_url),
}
: null,
fields:
embed.fields && embed.fields.length > 0
? embed.fields.map((field) => ({
name: field.name ?? null,
value: field.value ?? null,
inline: field.inline ?? false,
}))
: null,
html: embed.html ?? null,
html_width: embed.html_width ?? null,
html_height: embed.html_height ?? null,
nsfw: embed.nsfw ?? null,
};
}
private mapResponseMedia(media?: MessageEmbedResponse['image']): MessageEmbed['image'] {
if (!media) return null;
return {
url: stripOwnAttachmentSignature(media.url),
content_type: media.content_type ?? null,
content_hash: media.content_hash ?? null,
width: media.width ?? null,
height: media.height ?? null,
description: media.description ?? null,
placeholder: media.placeholder ?? null,
duration: media.duration ?? null,
flags: media.flags,
};
}
private validateEmbedSize(embeds: Array<RichEmbedRequest>, isBugHunterBot: boolean = false): void {
const totalChars = embeds.reduce<number>((sum, embed) => {
return (
sum +
(embed.title?.length || 0) +
(embed.description?.length || 0) +
(embed.footer?.text?.length || 0) +
(embed.author?.name?.length || 0) +
(embed.fields?.reduce((fieldSum, field) => fieldSum + field.name.length + field.value.length, 0) || 0)
);
}, 0);
const maxCharacters = isBugHunterBot ? this.MAX_EMBED_CHARACTERS_BUG_HUNTER : this.MAX_EMBED_CHARACTERS;
if (totalChars > maxCharacters) {
throw InputValidationError.fromCode('embeds', ValidationErrorCodes.EMBEDS_EXCEED_MAX_CHARACTERS, {
maxCharacters,
});
}
}
private async createEmbed(embed: RichEmbedRequestWithMedia, nsfwMode: MediaProxyNsfwMode): Promise<Embed> {
const [author, footer, imageResult, thumbnailResult] = await Promise.all([
this.processAuthor(embed.author ?? undefined, nsfwMode),
this.processFooter(embed.footer ?? undefined, nsfwMode),
this.processMedia(embed.image ?? undefined, nsfwMode),
this.processMedia(embed.thumbnail ?? undefined, nsfwMode),
]);
let nsfw: boolean | null = null;
const hasNSFWImage = imageResult?.nsfw ?? false;
const hasNSFWThumbnail = thumbnailResult?.nsfw ?? false;
if (hasNSFWImage || hasNSFWThumbnail) {
nsfw = true;
}
return new Embed({
type: 'rich',
title: embed.title ?? null,
description: embed.description ?? null,
url: canonicalUrl(embed.url),
timestamp: embed.timestamp ?? null,
color: embed.color ?? 0,
footer: footer?.toMessageEmbedFooter() ?? null,
image: imageResult?.media?.toMessageEmbedMedia() ?? null,
thumbnail: thumbnailResult?.media?.toMessageEmbedMedia() ?? null,
video: null,
provider: null,
author: author?.toMessageEmbedAuthor() ?? null,
fields:
embed.fields?.map((field) =>
new EmbedField({
name: field.name,
value: field.value,
inline: field.inline ?? false,
}).toMessageEmbedField(),
) ?? null,
children: null,
nsfw,
});
}
private async processMedia(
request?: RichEmbedMediaRequest | RichEmbedMediaWithMetadata,
nsfwMode: MediaProxyNsfwMode = 'block',
): Promise<{
media: EmbedMedia;
nsfw: boolean;
} | null> {
if (!request?.url) return null;
if (request.url.startsWith('attachment://')) {
throw InputValidationError.fromCode('embeds', ValidationErrorCodes.UNRESOLVED_ATTACHMENT_URL);
}
const attachmentMetadata = (request as RichEmbedMediaWithMetadata)._attachmentMetadata;
if (attachmentMetadata) {
return {
media: new EmbedMedia({
url: stripOwnAttachmentSignature(request.url),
width: attachmentMetadata.width,
height: attachmentMetadata.height,
description: request.description ?? null,
content_type: attachmentMetadata.content_type,
content_hash: attachmentMetadata.content_hash,
placeholder: attachmentMetadata.placeholder,
flags: attachmentMetadata.flags,
duration: attachmentMetadata.duration,
}),
nsfw: attachmentMetadata.nsfw ?? false,
};
}
const {url, metadata} = await this.resolveExternalMedia(stripOwnAttachmentSignature(request.url), nsfwMode);
if (!metadata) {
return {
media: new EmbedMedia({
url,
width: null,
height: null,
description: request.description ?? null,
content_type: null,
content_hash: null,
placeholder: null,
flags: 0,
duration: null,
}),
nsfw: false,
};
}
return {
media: new EmbedMedia({
url,
width: metadata.width ?? null,
height: metadata.height ?? null,
description: request.description ?? null,
content_type: metadata.content_type ?? null,
content_hash: metadata.content_hash ?? null,
placeholder: metadata.placeholder ?? null,
flags:
(metadata.animated ? EmbedMediaFlags.IS_ANIMATED : 0) |
(metadata.nsfw ? EmbedMediaFlags.CONTAINS_EXPLICIT_MEDIA : 0),
duration: metadata.duration ?? null,
}),
nsfw: metadata.nsfw,
};
}
private async resolveExternalMedia(
url: string,
nsfwMode: MediaProxyNsfwMode,
): Promise<{
url: string;
metadata: MediaProxyMetadataResponse | null;
}> {
const stored = stripOwnAttachmentSignature(url);
const directMetadata = await this.mediaService.getMetadata({
type: 'external',
url,
...mediaProxyMetadataPolicy(nsfwMode),
});
if (this.isRenderableMediaType(directMetadata?.content_type)) {
return {url: stored, metadata: directMetadata};
}
const unfurled = await this.unfurlerService.unfurl(url, nsfwMode).catch((error: unknown) => {
Logger.warn({error, url}, 'Failed to unfurl embed media URL, keeping it without metadata');
return [];
});
for (const embed of unfurled) {
const candidates = [embed.video?.url, embed.audio?.url, embed.image?.url, embed.thumbnail?.url];
for (const candidate of candidates) {
if (!candidate || candidate === url) continue;
const candidateMetadata = await this.mediaService.getMetadata({
type: 'external',
url: candidate,
...mediaProxyMetadataPolicy(nsfwMode),
});
if (this.isRenderableMediaType(candidateMetadata?.content_type)) {
return {url: stripOwnAttachmentSignature(candidate), metadata: candidateMetadata};
}
}
}
return {url: stored, metadata: directMetadata};
}
private isRenderableMediaType(contentType: string | null | undefined): boolean {
if (!contentType) return false;
return contentType.startsWith('image/') || contentType.startsWith('video/');
}
private async processAuthor(
author?: RichEmbedAuthorRequest,
nsfwMode: MediaProxyNsfwMode = 'block',
): Promise<EmbedAuthor | null> {
if (!author) return null;
let iconUrl: string | null = null;
if (author.icon_url) {
const metadata = await this.mediaService.getMetadata({
type: 'external',
url: author.icon_url,
...mediaProxyMetadataPolicy(nsfwMode),
});
if (metadata) iconUrl = stripOwnAttachmentSignature(author.icon_url);
}
return new EmbedAuthor({
name: author.name,
url: canonicalUrl(author.url),
icon_url: iconUrl,
});
}
private async processFooter(
footer?: RichEmbedFooterRequest,
nsfwMode: MediaProxyNsfwMode = 'block',
): Promise<EmbedFooter | null> {
if (!footer) return null;
let iconUrl: string | null = null;
if (footer.icon_url) {
const metadata = await this.mediaService.getMetadata({
type: 'external',
url: footer.icon_url,
...mediaProxyMetadataPolicy(nsfwMode),
});
if (metadata) iconUrl = stripOwnAttachmentSignature(footer.icon_url);
}
return new EmbedFooter({
text: footer.text,
icon_url: iconUrl,
});
}
private async enqueue(
channelId: ChannelID,
messageId: MessageID,
guildId: bigint | null,
nsfwMode: MediaProxyNsfwMode,
options: {content?: string | null} = {},
): Promise<void> {
const expectedContentHash =
options.content !== undefined ? UnfurlerUtils.hashUnfurlContent(options.content) : undefined;
try {
await this.workerService.addJob(
'extractEmbeds',
{
guildId: guildId ? guildId.toString() : null,
channelId: channelId.toString(),
messageId: messageId.toString(),
nsfwMode,
...(expectedContentHash ? {expectedContentHash} : {}),
},
{
jobKey: expectedContentHash ? `${messageId.toString()}:${expectedContentHash}` : messageId.toString(),
skipLedger: true,
},
);
} catch (error) {
if (!(error instanceof WorkerQueueOverflowError)) {
throw error;
}
Logger.warn(
{channelId: channelId.toString(), messageId: messageId.toString()},
'Dropped url embed extraction, jobs stream is at its limit',
);
}
}
}