A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
feat(api/worker): add pass data to data exports, share mappers
Hampus20bcf15dda3e
diff --git a/fluxer_api/src/api/worker/tasks/HarvestUserData.ts b/fluxer_api/src/api/worker/tasks/HarvestUserData.tsindex 1b24bfbcd..738931f07 100644--- a/fluxer_api/src/api/worker/tasks/HarvestUserData.ts+++ b/fluxer_api/src/api/worker/tasks/HarvestUserData.ts@@ -31,33 +31,19 @@ import {type SelfMessageEligibilityContext,type SelfMessageFilter,} from '@app/api/channel/services/message/SelfMessageFilter';-import type {UserConnectionRow} from '@app/api/database/types/ConnectionTypes';-import type {StorePurchaseRow} from '@app/api/database/types/StoreBillingTypes';+import type {OperatorPassUnitRow} from '@app/api/database/types/OperatorPassTypes';import {isTainted} from '@app/api/experiment/ChannelThreadsGate';import type {IStorageService} from '@app/api/infrastructure/IStorageService';import {Logger} from '@app/api/Logger';+import {getOperatorPassRepository, getOperatorPassWaitlistService} from '@app/api/middleware/ServiceSingletons';import type {Application} from '@app/api/models/Application';import type {Attachment} from '@app/api/models/Attachment';-import type {AuthSession} from '@app/api/models/AuthSession';import type {Channel} from '@app/api/models/Channel';-import type {FavoriteMeme} from '@app/api/models/FavoriteMeme';-import type {GiftCode} from '@app/api/models/GiftCode';-import type {Guild} from '@app/api/models/Guild';-import type {GuildMember} from '@app/api/models/GuildMember';-import type {MfaBackupCode} from '@app/api/models/MfaBackupCode';import type {Payment} from '@app/api/models/Payment';-import type {PushSubscription} from '@app/api/models/PushSubscription';-import type {Relationship} from '@app/api/models/Relationship';-import type {SavedMessage} from '@app/api/models/SavedMessage';import type {ThreadMember} from '@app/api/models/ThreadMember';import type {User} from '@app/api/models/User';-import type {UserGuildSettings} from '@app/api/models/UserGuildSettings';-import type {UserSettings} from '@app/api/models/UserSettings';-import type {WebAuthnCredential} from '@app/api/models/WebAuthnCredential';-import {mapStorePurchaseToResponse} from '@app/api/store_billing/StoreBillingMappers';import {buildHarvestDownloadUrl} from '@app/api/user/services/HarvestDownloadUrl';import {mapWithConcurrency} from '@app/api/utils/ConcurrencyUtils';-import {resolveSessionClientInfo} from '@app/api/utils/SessionClientIdentity';import {writeZipArchive} from '@app/api/worker/utils/ArchiveFile';import {createArchiveJsonBuffer} from '@app/api/worker/utils/ArchiveJson';import {@@ -66,13 +52,16 @@ import {getAnimatedAssetExtension,} from '@app/api/worker/utils/AssetArchiveHelpers';import {ContentAddressedAttachmentCollector} from '@app/api/worker/utils/ContentAddressedAttachmentCollector';+import {+ buildUserDataJson,+ mapOAuthApplication,+ mapPayment,+ mapSecurityData,+} from '@app/api/worker/utils/HarvestDataMappers';import {deserializeSelfMessageFilter, SelfMessageFilterPayload} from '@app/api/worker/utils/SelfMessageFilterPayload';import {getWorkerDependencies} from '@app/api/worker/WorkerContext';import type {WorkerDependencies} from '@app/api/worker/WorkerDependencies';-import {- decodeSyncedPreferencesLenient,- syncedPreferencesToJson,-} from '@fluxer/schema/src/domains/user/SyncedPreferencesCodec';+import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';import {ms} from 'itty-time';import {z} from 'zod';@@ -109,53 +98,11 @@ interface ChannelHarvestResult {messageData: HarvestedMessage;}-interface GuildMembershipEntry {- member: GuildMember | null;- guild: Guild | null;- guildId: GuildID;-}-interface HarvestMessageResult {channelMessagesMap: Map<string, Array<HarvestedMessage>>;totalMessages: number;}-interface UserDataJsonParams {- user: User;- userId: UserID;- productName: string;- authSessions: Array<AuthSession>;- relationships: Array<Relationship>;- userNotes: Map<UserID, string>;- userSettings: UserSettings | null;- guildMemberships: Array<GuildMembershipEntry>;- guildSettings: Array<UserGuildSettings | null>;- threadMemberships: Array<ThreadMember>;- savedMessages: Array<SavedMessage>;- privateChannels: Array<Channel>;- favoriteMemes: Array<FavoriteMeme>;- pushSubscriptions: Array<PushSubscription>;- webAuthnCredentials: Array<WebAuthnCredential>;- mfaBackupCodes: Array<MfaBackupCode>;- recoveryKitCreatedAt: Date | null;- createdGiftCodes: Array<GiftCode>;- payments: Array<Payment>;- storePurchases: Array<StorePurchaseRow>;- oauthClients: Array<Application>;- connections: Array<UserConnectionRow>;- pinnedDms: Array<{- channel_id: bigint;- sort_order: number;- }>;- authorizedIps: Array<{- ip: string;- }>;- activityData: {- last_active_at: Date | null;- last_active_ip: string | null;- };-}-interface ArchiveParams {userId: UserID;harvestId: bigint;@@ -238,9 +185,6 @@ async function claimUserArchive(};}-// A harvest is every message the account wrote, so the read pages to the end of-// the account rather than stopping at a count. The page size is what bounds one-// query, not what bounds the archive.const HARVEST_MESSAGE_CHUNK_SIZE = 1000;const HARVEST_READ_CONCURRENCY = 10;const INITIAL_PROGRESS = 5;@@ -248,48 +192,6 @@ const MESSAGES_PROGRESS_MAX = 55;const METADATA_PROGRESS = 60;const ZIP_EXPIRY_MS = ms('7 days');-function mapPayment(payment: Payment) {- return {- checkout_session_id: payment.checkoutSessionId,- amount_cents: payment.amountCents,- currency: payment.currency,- status: payment.status,- subscription_id: payment.subscriptionId,- payment_intent_id: payment.paymentIntentId,- product_type: payment.productType,- is_gift: payment.isGift,- gift_code: payment.giftCode,- created_at: payment.createdAt.toISOString(),- completed_at: payment.completedAt?.toISOString() ?? null,- };-}--function mapOAuthApplication(app: Application) {- return {- application_id: app.applicationId.toString(),- name: app.name,- redirect_uris: Array.from(app.oauth2RedirectUris),- };-}--function mapSecurityData(params: {- authorizedIps: Array<{- ip: string;- }>;- activityData: {- last_active_at: Date | null;- last_active_ip: string | null;- };-}) {- return {- authorized_ips: params.authorizedIps,- activity_tracking: {- last_active_at: params.activityData.last_active_at?.toISOString() ?? null,- last_active_ip: params.activityData.last_active_ip,- },- };-}-interface HarvestMessagesFilterArgs {filter: SelfMessageFilter;context: SelfMessageEligibilityContext;@@ -313,14 +215,14 @@ interface HarvestMessageRepository {): Promise<{content: string | null; attachments?: Array<Attachment>} | null>;}-export async function harvestMessages(+async function harvestMessages(channelRepository: HarvestMessageRepository,userId: UserID,startTime: number,filterArgs: HarvestMessagesFilterArgs | null,): Promise<HarvestMessageResult> {const channelMessagesMap = new Map<string, Array<HarvestedMessage>>();- const channelEligibility = filterArgs ? new Map<string, boolean>() : null;+ const channelEligibility = new Map<string, boolean>();Logger.debug('Fetching all user messages');const startFetchTime = Date.now();let lastMessageId: MessageID | undefined;@@ -376,11 +278,11 @@ export async function harvestMessages(continue;}const channelIdStr = ref.channelId.toString();- let eligible = channelEligibility!.get(channelIdStr);+ let eligible = channelEligibility.get(channelIdStr);if (eligible === undefined) {const channel = await filterArgs.findChannel(ref.channelId);eligible = channel ? isChannelEligible(channel, userId, filterArgs.filter, filterArgs.context) : false;- channelEligibility!.set(channelIdStr, eligible);+ channelEligibility.set(channelIdStr, eligible);}if (eligible) {pageRefs.push(ref);@@ -416,7 +318,7 @@ export async function harvestMessages(return {channelMessagesMap, totalMessages};}-export async function collectThreadMemberships(+async function collectThreadMemberships(threads: Pick<ThreadRepository, 'listJoinedThreadIds' | 'getMember'>,userId: UserID,guildIds: ReadonlyArray<GuildID>,@@ -430,268 +332,6 @@ export async function collectThreadMemberships(return perGuild.flat();}-export function buildUserDataJson(params: UserDataJsonParams) {- const {- user,- userId,- productName,- authSessions,- relationships,- userNotes,- userSettings,- guildMemberships,- guildSettings,- threadMemberships,- savedMessages,- privateChannels,- favoriteMemes,- pushSubscriptions,- webAuthnCredentials,- mfaBackupCodes,- recoveryKitCreatedAt,- createdGiftCodes,- payments,- storePurchases,- oauthClients,- connections,- pinnedDms,- authorizedIps,- activityData,- } = params;- return {- user: {- id: user.id.toString(),- username: user.username,- discriminator: user.discriminator,- bot: user.isBot,- system: user.isSystem,- email: user.email,- email_verified: user.emailVerified,- email_bounced: user.emailBounced,- avatar_hash: user.avatarHash,- avatar_url: user.avatarHash- ? `${Config.endpoints.media}/avatars/${userId}/${user.avatarHash}.${getAnimatedAssetExtension(user.avatarHash)}`- : null,- banner_hash: user.bannerHash,- banner_url: user.bannerHash- ? `${Config.endpoints.media}/banners/${userId}/${user.bannerHash}.${getAnimatedAssetExtension(user.bannerHash)}`- : null,- bio: user.bio,- pronouns: user.pronouns,- accent_color: user.accentColor,- date_of_birth: user.dateOfBirth,- locale: user.locale,- flags: user.flags.toString(),- premium_type: user.premiumType,- premium_since: user.premiumSince?.toISOString() ?? null,- premium_until: user.premiumUntil?.toISOString() ?? null,- premium_lifetime_sequence: user.premiumLifetimeSequence,- stripe_customer_id: user.stripeCustomerId,- stripe_subscription_id: user.stripeSubscriptionId,- terms_agreed_at: user.termsAgreedAt?.toISOString() ?? null,- privacy_agreed_at: user.privacyAgreedAt?.toISOString() ?? null,- last_active_at: user.lastActiveAt?.toISOString() ?? null,- created_at: snowflakeToDate(user.id).toISOString(),- mfa_enabled: user.authenticatorTypes.size > 0,- authenticator_types: Array.from(user.authenticatorTypes),- },- auth_sessions: authSessions.map((session) => {- const clientInfo = resolveSessionClientInfo({- userAgent: session.clientUserAgent,- reportedOs: session.clientOs ?? null,- productName,- });- return {- created_at: session.createdAt.toISOString(),- approx_last_used_at: session.approximateLastUsedAt?.toISOString() ?? null,- client_ip: session.clientIp,- client_os: clientInfo.os,- client_user_agent: session.clientUserAgent,- client_platform: clientInfo.platform,- };- }),- relationships: relationships.map((rel) => ({- target_user_id: rel.targetUserId.toString(),- type: rel.type,- nickname: rel.nickname,- since: rel.since?.toISOString() ?? null,- })),- notes: Array.from(userNotes.entries()).map(([targetUserId, note]) => ({- target_user_id: targetUserId.toString(),- note,- })),- user_settings: userSettings- ? {- locale: userSettings.locale,- theme: userSettings.theme,- status: userSettings.status,- custom_status: userSettings.customStatus- ? {- text: userSettings.customStatus.text,- emoji_id: userSettings.customStatus.emojiId?.toString() ?? null,- emoji_name: userSettings.customStatus.emojiName,- emoji_animated: userSettings.customStatus.emojiAnimated,- expires_at: userSettings.customStatus.expiresAt?.toISOString() ?? null,- }- : null,- developer_mode: userSettings.developerMode,- message_display_compact: userSettings.compactMessageDisplay,- animate_emoji: userSettings.animateEmoji,- animate_stickers: userSettings.animateStickers,- gif_auto_play: userSettings.gifAutoPlay,- render_embeds: userSettings.renderEmbeds,- render_reactions: userSettings.renderReactions,- render_spoilers: userSettings.renderSpoilers,- inline_attachment_media: userSettings.inlineAttachmentMedia,- inline_embed_media: userSettings.inlineEmbedMedia,- explicit_content_filter: userSettings.explicitContentFilter,- friend_source_flags: userSettings.friendSourceFlags,- default_guilds_restricted: userSettings.defaultGuildsRestricted,- bot_default_guilds_restricted: userSettings.botDefaultGuildsRestricted,- restricted_guilds: Array.from(userSettings.restrictedGuilds).map((id) => id.toString()),- bot_restricted_guilds: Array.from(userSettings.botRestrictedGuilds).map((id) => id.toString()),- guild_positions: userSettings.guildPositions.map((id) => id.toString()),- guild_folders: userSettings.guildFolders.map((folder) => ({- id: folder.folderId,- name: folder.name,- color: folder.color,- flags: folder.flags,- icon: folder.icon,- guild_ids: folder.guildIds.map(String),- })),- afk_timeout: userSettings.afkTimeout,- time_format: userSettings.timeFormat,- suppress_unprivileged_self_mentions: userSettings.suppressUnprivilegedSelfMentions,- suppress_unprivileged_self_mentions_bypass_user_ids: Array.from(- userSettings.suppressUnprivilegedSelfMentionBypassUserIds,- ).map((id) => id.toString()),- staff_dm_access_user_ids: Array.from(userSettings.staffDmAccessUserIds).map((id) => id.toString()),- synced_preferences: syncedPreferencesToJson(decodeSyncedPreferencesLenient(userSettings.syncedPreferences)),- profile_privacy: userSettings.profilePrivacy,- privacy_setup_version: userSettings.privacySetupVersion,- privacy_setup_completed_at: userSettings.privacySetupCompletedAt?.toISOString() ?? null,- }- : null,- guild_memberships: guildMemberships- .filter((gm) => gm.member !== null)- .map(({member, guild, guildId}) => ({- guild_id: guildId.toString(),- guild_name: guild?.name ?? null,- joined_at: member!.joinedAt.toISOString(),- nick: member!.nickname,- avatar_hash: member!.avatarHash,- avatar_url: member!.avatarHash- ? `${Config.endpoints.media}/guilds/${guildId}/users/${userId}/avatars/${member!.avatarHash}`- : null,- banner_hash: member!.bannerHash,- banner_url: member!.bannerHash- ? `${Config.endpoints.media}/guilds/${guildId}/users/${userId}/banners/${member!.bannerHash}`- : null,- role_ids: Array.from(member!.roleIds).map((id) => id.toString()),- })),- user_guild_settings: guildSettings- .filter((settings) => settings !== null)- .map((settings) => ({- guild_id: settings!.guildId.toString(),- message_notifications: settings!.messageNotifications,- muted: settings!.muted,- mobile_push: settings!.mobilePush,- suppress_everyone: settings!.suppressEveryone,- suppress_roles: settings!.suppressRoles,- hide_muted_channels: settings!.hideMutedChannels,- })),- ...(threadMemberships.length > 0- ? {- thread_memberships: threadMemberships.map((member) => ({- thread_id: member.threadId.toString(),- guild_id: member.guildId.toString(),- parent_id: member.parentId.toString(),- join_timestamp: member.joinTimestamp.toISOString(),- flags: member.flags,- muted: member.muted,- })),- }- : {}),- saved_messages: savedMessages.map((msg) => ({- channel_id: msg.channelId.toString(),- message_id: msg.messageId.toString(),- saved_at: msg.savedAt.toISOString(),- })),- private_channels: privateChannels.map((channel) => ({- channel_id: channel.id.toString(),- type: channel.type,- name: channel.name,- icon_hash: channel.iconHash,- owner_id: channel.ownerId?.toString() ?? null,- recipient_ids: Array.from(channel.recipientIds).map((id) => id.toString()),- last_message_id: channel.lastMessageId?.toString() ?? null,- })),- favorite_memes: favoriteMemes.map((meme) => ({- meme_id: meme.id.toString(),- name: meme.name,- alt_text: meme.altText,- tags: meme.tags,- filename: meme.filename,- content_type: meme.contentType,- size: meme.size.toString(),- width: meme.width,- height: meme.height,- duration: meme.duration,- })),- push_subscriptions: pushSubscriptions.map((sub) => ({- subscription_id: sub.subscriptionId,- platform: sub.platform,- app_id: sub.appId,- provider_environment: sub.providerEnvironment,- endpoint: sub.endpoint,- user_agent: sub.userAgent,- })),- webauthn_credentials: webAuthnCredentials.map((cred) => ({- credential_id: cred.credentialId,- name: cred.name,- transports: cred.transports ? Array.from(cred.transports) : [],- created_at: cred.createdAt.toISOString(),- last_used_at: cred.lastUsedAt?.toISOString() ?? null,- })),- mfa_backup_codes: {- total_count: mfaBackupCodes.length,- consumed_count: mfaBackupCodes.filter((code) => code.consumed).length,- remaining_count: mfaBackupCodes.filter((code) => !code.consumed).length,- },- ...(recoveryKitCreatedAt ? {recovery_kit: {created_at: recoveryKitCreatedAt.toISOString()}} : {}),- gift_codes_created: createdGiftCodes.map((gift) => ({- code: gift.code,- duration_months: gift.durationMonths,- duration_type: gift.durationType,- duration_quantity: gift.durationQuantity,- created_at: gift.createdAt.toISOString(),- redeemed_by_user_id: gift.redeemedByUserId?.toString() ?? null,- redeemed_at: gift.redeemedAt?.toISOString() ?? null,- stripe_payment_intent_id: gift.stripePaymentIntentId,- })),- payments: payments.map(mapPayment),- store_purchases: storePurchases.map(mapStorePurchaseToResponse),- oauth_applications: oauthClients.map(mapOAuthApplication),- connections: connections.map((connection) => ({- id: connection.connection_id,- type: connection.connection_type,- identifier: connection.identifier,- name: connection.name,- verified: connection.verified,- visibility_flags: connection.visibility_flags,- sort_order: connection.sort_order,- created_at: connection.created_at.toISOString(),- verified_at: connection.verified_at?.toISOString() ?? null,- last_verified_at: connection.last_verified_at?.toISOString() ?? null,- })),- pinned_dms: pinnedDms.map((pin) => ({- channel_id: pin.channel_id.toString(),- sort_order: pin.sort_order,- })),- ...mapSecurityData({authorizedIps, activityData}),- };-}-async function createAndUploadArchive(params: ArchiveParams): Promise<ArchiveResult> {const {userId,@@ -882,6 +522,11 @@ const harvestUserData: ArchiveTaskHandler = async (payload, helpers, attempt) =>createdGiftCodes,payments,storePurchases,+ operatorPass,+ operatorPassPurchases,+ operatorPassWaitlistEntry,+ operatorPassWall,+ operatorPassWallEvents,oauthClients,connections,pinnedDms,@@ -903,12 +548,34 @@ const harvestUserData: ArchiveTaskHandler = async (payload, helpers, attempt) =>userRepository.findGiftCodesByCreator(userId),paymentRepository.findPaymentsByUserId(userId),storeEntitlementService.listStorePurchases(userId),+ getOperatorPassRepository().findOperatorPass(userId),+ getOperatorPassRepository().listPurchasesByPurchaser(userId),+ user.email ? getOperatorPassWaitlistService().findEntryForAddress(user.email) : null,+ getOperatorPassRepository().findWallEntry(userId),+ getOperatorPassRepository().listWallEvents(userId),applicationRepository.listApplicationsByOwner(userId),connectionRepository.findByUserId(userId),userRepository.getPinnedDmsWithDetails(userId),userRepository.getAuthorizedIps(userId),userRepository.getActivityTracking(userId),]);+ const operatorPassRepository = getOperatorPassRepository();+ const operatorPassUnitRows = new Map<string, OperatorPassUnitRow>();+ for (const unit of await operatorPassRepository.listUnitsByHolder(userId)) {+ operatorPassUnitRows.set(unit.pass_id, unit);+ }+ for (const purchase of operatorPassPurchases) {+ for (const unit of await operatorPassRepository.listUnitsByPurchase(purchase.checkout_session_id)) {+ operatorPassUnitRows.set(unit.pass_id, unit);+ }+ }+ const operatorPassUnits = await Promise.all(+ [...operatorPassUnitRows.values()].map(async (unit) => ({+ unit,+ events: await operatorPassRepository.listUnitEvents(unit.pass_id),+ })),+ );+ const operatorPassOffers = await operatorPassRepository.listOffersByUser(userId);const guilds = await guildRepository.listGuilds(guildIds);const guildsMap = new Map(guilds.map((guild) => [guild.id.toString(), guild]));const guildMemberships = await mapWithConcurrency(guildIds, HARVEST_READ_CONCURRENCY, async (guildId) => {@@ -942,6 +609,13 @@ const harvestUserData: ArchiveTaskHandler = async (payload, helpers, attempt) =>createdGiftCodes,payments,storePurchases,+ operatorPass,+ operatorPassPurchases,+ operatorPassWaitlistEntry,+ operatorPassWall,+ operatorPassWallEvents,+ operatorPassUnits,+ operatorPassOffers,oauthClients,connections,pinnedDms,@@ -1003,13 +677,15 @@ const harvestUserData: ArchiveTaskHandler = async (payload, helpers, attempt) =>await progressReporter.markAsCompleted(archive.storageKey, archive.fileSize, archive.expiresAt);Logger.debug({userId, harvestId}, 'Marked harvest as completed');if (!isAdminArchive && user.email && (await instanceConfigRepository.isEmailEnabled())) {- const sent = await emailService.sendHarvestCompletedEmail(+ const sent = await emailService.send(+ 'harvest_completed',user.email,- user.username,- archive.downloadUrl,- totalMessages,- Number(archive.fileSize),- archive.expiresAt,+ {+ downloadUrl: archive.downloadUrl,+ totalMessages,+ fileSize: Number(archive.fileSize),+ expiresAt: archive.expiresAt,+ },user.locale,);if (!sent) throw new Error(`Completion email for harvest ${harvestId} was not sent`);diff --git a/fluxer_api/src/api/worker/utils/HarvestDataMappers.ts b/fluxer_api/src/api/worker/utils/HarvestDataMappers.tsnew file mode 100644index 000000000..80e90a455--- /dev/null+++ b/fluxer_api/src/api/worker/utils/HarvestDataMappers.ts@@ -0,0 +1,613 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import type {GuildID, UserID} from '@app/api/BrandedTypes';+import {Config} from '@app/api/Config';+import type {UserConnectionRow} from '@app/api/database/types/ConnectionTypes';+import type {+ OperatorPassOfferRow,+ OperatorPassPurchaseRow,+ OperatorPassRow,+ OperatorPassUnitEventRow,+ OperatorPassUnitRow,+ OperatorPassWaitlistRow,+ OperatorPassWallEntryRow,+ OperatorPassWallEventRow,+} from '@app/api/database/types/OperatorPassTypes';+import type {StorePurchaseRow} from '@app/api/database/types/StoreBillingTypes';+import type {Application} from '@app/api/models/Application';+import type {AuthSession} from '@app/api/models/AuthSession';+import type {Channel} from '@app/api/models/Channel';+import type {FavoriteMeme} from '@app/api/models/FavoriteMeme';+import type {GiftCode} from '@app/api/models/GiftCode';+import type {Guild} from '@app/api/models/Guild';+import type {GuildMember} from '@app/api/models/GuildMember';+import type {MfaBackupCode} from '@app/api/models/MfaBackupCode';+import type {Payment} from '@app/api/models/Payment';+import type {PushSubscription} from '@app/api/models/PushSubscription';+import type {Relationship} from '@app/api/models/Relationship';+import type {SavedMessage} from '@app/api/models/SavedMessage';+import type {ThreadMember} from '@app/api/models/ThreadMember';+import type {User} from '@app/api/models/User';+import type {UserGuildSettings} from '@app/api/models/UserGuildSettings';+import type {UserSettings} from '@app/api/models/UserSettings';+import type {WebAuthnCredential} from '@app/api/models/WebAuthnCredential';+import {mapStorePurchaseToResponse} from '@app/api/store_billing/StoreBillingMappers';+import {resolveSessionClientInfo} from '@app/api/utils/SessionClientIdentity';+import {getAnimatedAssetExtension} from '@app/api/worker/utils/AssetArchiveHelpers';+import {+ decodeSyncedPreferencesLenient,+ syncedPreferencesToJson,+} from '@fluxer/schema/src/domains/user/SyncedPreferencesCodec';+import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';++interface GuildMembershipEntry {+ member: GuildMember | null;+ guild: Guild | null;+ guildId: GuildID;+}++interface ArchivedUserSettings {+ locale: UserSettings['locale'];+ theme: string;+ status: string;+ custom_status: {+ text: string | null;+ emoji_id: string | null;+ emoji_name: string | null;+ emoji_animated: boolean;+ expires_at: string | null;+ } | null;+ developer_mode: boolean;+ message_display_compact: boolean;+ animate_emoji: boolean;+ animate_stickers: UserSettings['animateStickers'];+ gif_auto_play: boolean;+ render_embeds: boolean;+ render_reactions: boolean;+ render_spoilers: UserSettings['renderSpoilers'];+ inline_attachment_media: boolean;+ inline_embed_media: boolean;+ explicit_content_filter: number;+ friend_source_flags: number;+ default_guilds_restricted: boolean;+ bot_default_guilds_restricted: boolean;+ restricted_guilds: Array<string>;+ bot_restricted_guilds: Array<string>;+ guild_positions: Array<string>;+ guild_folders: Array<{+ id: number;+ name: string | null;+ color: number | null;+ flags: number;+ icon: UserSettings['guildFolders'][number]['icon'];+ guild_ids: Array<string>;+ }>;+ afk_timeout: number;+ time_format: UserSettings['timeFormat'];+ suppress_unprivileged_self_mentions: boolean;+ suppress_unprivileged_self_mentions_bypass_user_ids: Array<string>;+ staff_dm_access_user_ids: Array<string>;+ synced_preferences: unknown;+ profile_privacy: UserSettings['profilePrivacy'];+ privacy_setup_version: number;+ privacy_setup_completed_at: string | null;+}++interface UserDataJsonParams {+ user: User;+ userId: UserID;+ productName: string;+ authSessions: Array<AuthSession>;+ relationships: Array<Relationship>;+ userNotes: Map<UserID, string>;+ userSettings: UserSettings | null;+ guildMemberships: Array<GuildMembershipEntry>;+ guildSettings: Array<UserGuildSettings | null>;+ threadMemberships: Array<ThreadMember>;+ savedMessages: Array<SavedMessage>;+ privateChannels: Array<Channel>;+ favoriteMemes: Array<FavoriteMeme>;+ pushSubscriptions: Array<PushSubscription>;+ webAuthnCredentials: Array<WebAuthnCredential>;+ mfaBackupCodes: Array<MfaBackupCode>;+ recoveryKitCreatedAt: Date | null;+ createdGiftCodes: Array<GiftCode>;+ payments: Array<Payment>;+ storePurchases: Array<StorePurchaseRow>;+ operatorPass: OperatorPassRow | null;+ operatorPassPurchases: Array<OperatorPassPurchaseRow>;+ operatorPassWaitlistEntry: OperatorPassWaitlistRow | null;+ operatorPassWall: OperatorPassWallEntryRow | null;+ operatorPassWallEvents: Array<OperatorPassWallEventRow>;+ operatorPassUnits: Array<{unit: OperatorPassUnitRow; events: Array<OperatorPassUnitEventRow>}>;+ operatorPassOffers: Array<OperatorPassOfferRow>;+ oauthClients: Array<Application>;+ connections: Array<UserConnectionRow>;+ pinnedDms: Array<{+ channel_id: bigint;+ sort_order: number;+ }>;+ authorizedIps: Array<{+ ip: string;+ }>;+ activityData: {+ last_active_at: Date | null;+ last_active_ip: string | null;+ };+}++function mapOperatorPassUnit(unit: OperatorPassUnitRow, events: Array<OperatorPassUnitEventRow>, userId: UserID) {+ const bought = unit.buyer_id === userId;+ const handOff = events.findIndex((event) => event.kind === 'transferred');+ const received = events.findIndex((event) => event.kind === 'transferred' && event.recipient_id === userId);+ const passedOn = bought && handOff !== -1;+ let visible = events;+ if (passedOn) {+ visible = events.slice(0, handOff + 1);+ } else if (!bought && received !== -1) {+ visible = events.slice(received);+ } else if (!bought && unit.purchase_id !== null) {+ visible = [];+ }+ return {+ id: bought || unit.purchase_id === null ? unit.pass_id : null,+ purchase_id: bought ? (unit.purchase_id ?? null) : null,+ source: unit.source,+ status: passedOn ? 'passed_on' : unit.status,+ held_by_you: unit.holder_id === userId,+ delivered_at: (passedOn ? unit.first_delivered_at : unit.delivered_at).toISOString(),+ first_delivered_at: (bought ? unit.first_delivered_at : unit.delivered_at).toISOString(),+ ended_at: passedOn ? null : (unit.ended_at?.toISOString() ?? null),+ end_reason: passedOn ? null : (unit.end_reason ?? null),+ withdrawal_country: bought ? (unit.withdrawal_country ?? null) : null,+ withdrawal_days: bought ? (unit.withdrawal_days ?? null) : null,+ events: visible.map((event) => ({+ occurred_at: event.occurred_at.toISOString(),+ kind: event.kind,+ payment_id: bought ? (event.payment_id ?? null) : null,+ sender_id: event.sender_id?.toString() ?? null,+ recipient_id: event.recipient_id?.toString() ?? null,+ offer_id: event.offer_id ?? null,+ country: bought ? (event.country ?? null) : null,+ days: bought ? (event.days ?? null) : null,+ rule_version: bought ? (event.rule_version ?? null) : null,+ reason: event.reason ?? null,+ })),+ };+}++export function mapPayment(payment: Payment) {+ return {+ checkout_session_id: payment.checkoutSessionId,+ amount_cents: payment.amountCents,+ currency: payment.currency,+ status: payment.status,+ subscription_id: payment.subscriptionId,+ payment_intent_id: payment.paymentIntentId,+ product_type: payment.productType,+ is_gift: payment.isGift,+ gift_code: payment.giftCode,+ created_at: payment.createdAt.toISOString(),+ completed_at: payment.completedAt?.toISOString() ?? null,+ };+}++export function mapOAuthApplication(app: Application) {+ return {+ application_id: app.applicationId.toString(),+ name: app.name,+ redirect_uris: Array.from(app.oauth2RedirectUris),+ };+}++export function mapSecurityData(params: {+ authorizedIps: Array<{+ ip: string;+ }>;+ activityData: {+ last_active_at: Date | null;+ last_active_ip: string | null;+ };+}) {+ return {+ authorized_ips: params.authorizedIps,+ activity_tracking: {+ last_active_at: params.activityData.last_active_at?.toISOString() ?? null,+ last_active_ip: params.activityData.last_active_ip,+ },+ };+}++export function buildUserDataJson(params: UserDataJsonParams) {+ const {+ user,+ userId,+ productName,+ authSessions,+ relationships,+ userNotes,+ userSettings,+ guildMemberships,+ guildSettings,+ threadMemberships,+ savedMessages,+ privateChannels,+ favoriteMemes,+ pushSubscriptions,+ webAuthnCredentials,+ mfaBackupCodes,+ recoveryKitCreatedAt,+ createdGiftCodes,+ payments,+ storePurchases,+ operatorPass,+ operatorPassPurchases,+ operatorPassWaitlistEntry,+ operatorPassWall,+ operatorPassWallEvents,+ operatorPassUnits,+ operatorPassOffers,+ oauthClients,+ connections,+ pinnedDms,+ authorizedIps,+ activityData,+ } = params;+ const responseFields = {+ user: {+ id: user.id.toString(),+ username: user.username,+ discriminator: user.discriminator,+ bot: user.isBot,+ system: user.isSystem,+ email: user.email,+ email_verified: user.emailVerified,+ email_bounced: user.emailBounced,+ avatar_hash: user.avatarHash,+ avatar_url: user.avatarHash+ ? `${Config.endpoints.media}/avatars/${userId}/${user.avatarHash}.${getAnimatedAssetExtension(user.avatarHash)}`+ : null,+ banner_hash: user.bannerHash,+ banner_url: user.bannerHash+ ? `${Config.endpoints.media}/banners/${userId}/${user.bannerHash}.${getAnimatedAssetExtension(user.bannerHash)}`+ : null,+ bio: user.bio,+ pronouns: user.pronouns,+ accent_color: user.accentColor,+ date_of_birth: user.dateOfBirth,+ locale: user.locale,+ flags: user.flags.toString(),+ premium_type: user.premiumType,+ premium_since: user.premiumSince?.toISOString() ?? null,+ premium_until: user.premiumUntil?.toISOString() ?? null,+ premium_lifetime_sequence: user.premiumLifetimeSequence,+ stripe_customer_id: user.stripeCustomerId,+ stripe_subscription_id: user.stripeSubscriptionId,+ terms_agreed_at: user.termsAgreedAt?.toISOString() ?? null,+ privacy_agreed_at: user.privacyAgreedAt?.toISOString() ?? null,+ last_active_at: user.lastActiveAt?.toISOString() ?? null,+ created_at: snowflakeToDate(user.id).toISOString(),+ mfa_enabled: user.authenticatorTypes.size > 0,+ authenticator_types: Array.from(user.authenticatorTypes),+ },+ auth_sessions: authSessions.map((session) => {+ const clientInfo = resolveSessionClientInfo({+ userAgent: session.clientUserAgent,+ reportedOs: session.clientOs ?? null,+ productName,+ });+ return {+ created_at: session.createdAt.toISOString(),+ approx_last_used_at: session.approximateLastUsedAt?.toISOString() ?? null,+ client_ip: session.clientIp,+ client_os: clientInfo.os,+ client_user_agent: session.clientUserAgent,+ client_platform: clientInfo.platform,+ };+ }),+ relationships: relationships.map((rel) => ({+ target_user_id: rel.targetUserId.toString(),+ type: rel.type,+ nickname: rel.nickname,+ since: rel.since?.toISOString() ?? null,+ })),+ notes: Array.from(userNotes.entries()).map(([targetUserId, note]) => ({+ target_user_id: targetUserId.toString(),+ note,+ })),+ };+ let archivedSettings: ArchivedUserSettings | null = null;+ if (userSettings) {+ const statusFields = {+ locale: userSettings.locale,+ theme: userSettings.theme,+ status: userSettings.status,+ };+ let customStatus: ArchivedUserSettings['custom_status'] = null;+ if (userSettings.customStatus) {+ customStatus = {+ text: userSettings.customStatus.text,+ emoji_id: userSettings.customStatus.emojiId?.toString() ?? null,+ emoji_name: userSettings.customStatus.emojiName,+ emoji_animated: userSettings.customStatus.emojiAnimated,+ expires_at: userSettings.customStatus.expiresAt?.toISOString() ?? null,+ };+ }+ archivedSettings = {+ ...statusFields,+ custom_status: customStatus,+ developer_mode: userSettings.developerMode,+ message_display_compact: userSettings.compactMessageDisplay,+ animate_emoji: userSettings.animateEmoji,+ animate_stickers: userSettings.animateStickers,+ gif_auto_play: userSettings.gifAutoPlay,+ render_embeds: userSettings.renderEmbeds,+ render_reactions: userSettings.renderReactions,+ render_spoilers: userSettings.renderSpoilers,+ inline_attachment_media: userSettings.inlineAttachmentMedia,+ inline_embed_media: userSettings.inlineEmbedMedia,+ explicit_content_filter: userSettings.explicitContentFilter,+ friend_source_flags: userSettings.friendSourceFlags,+ default_guilds_restricted: userSettings.defaultGuildsRestricted,+ bot_default_guilds_restricted: userSettings.botDefaultGuildsRestricted,+ restricted_guilds: Array.from(userSettings.restrictedGuilds).map((id) => id.toString()),+ bot_restricted_guilds: Array.from(userSettings.botRestrictedGuilds).map((id) => id.toString()),+ guild_positions: userSettings.guildPositions.map((id) => id.toString()),+ guild_folders: userSettings.guildFolders.map((folder) => ({+ id: folder.folderId,+ name: folder.name,+ color: folder.color,+ flags: folder.flags,+ icon: folder.icon,+ guild_ids: folder.guildIds.map(String),+ })),+ afk_timeout: userSettings.afkTimeout,+ time_format: userSettings.timeFormat,+ suppress_unprivileged_self_mentions: userSettings.suppressUnprivilegedSelfMentions,+ suppress_unprivileged_self_mentions_bypass_user_ids: Array.from(+ userSettings.suppressUnprivilegedSelfMentionBypassUserIds,+ ).map((id) => id.toString()),+ staff_dm_access_user_ids: Array.from(userSettings.staffDmAccessUserIds).map((id) => id.toString()),+ synced_preferences: syncedPreferencesToJson(decodeSyncedPreferencesLenient(userSettings.syncedPreferences)),+ profile_privacy: userSettings.profilePrivacy,+ privacy_setup_version: userSettings.privacySetupVersion,+ privacy_setup_completed_at: userSettings.privacySetupCompletedAt?.toISOString() ?? null,+ };+ }+ return {+ ...responseFields,+ user_settings: archivedSettings,+ guild_memberships: guildMemberships+ .filter((membership): membership is GuildMembershipEntry & {member: GuildMember} => membership.member !== null)+ .map(({member, guild, guildId}) => ({+ guild_id: guildId.toString(),+ guild_name: guild?.name ?? null,+ joined_at: member.joinedAt.toISOString(),+ nick: member.nickname,+ avatar_hash: member.avatarHash,+ avatar_url: member.avatarHash+ ? `${Config.endpoints.media}/guilds/${guildId}/users/${userId}/avatars/${member.avatarHash}`+ : null,+ banner_hash: member.bannerHash,+ banner_url: member.bannerHash+ ? `${Config.endpoints.media}/guilds/${guildId}/users/${userId}/banners/${member.bannerHash}`+ : null,+ role_ids: Array.from(member.roleIds).map((id) => id.toString()),+ })),+ user_guild_settings: guildSettings+ .filter((settings) => settings !== null)+ .map((settings) => ({+ guild_id: settings.guildId.toString(),+ message_notifications: settings.messageNotifications,+ muted: settings.muted,+ mobile_push: settings.mobilePush,+ suppress_everyone: settings.suppressEveryone,+ suppress_roles: settings.suppressRoles,+ hide_muted_channels: settings.hideMutedChannels,+ })),+ ...(threadMemberships.length > 0+ ? {+ thread_memberships: threadMemberships.map((member) => ({+ thread_id: member.threadId.toString(),+ guild_id: member.guildId.toString(),+ parent_id: member.parentId.toString(),+ join_timestamp: member.joinTimestamp.toISOString(),+ flags: member.flags,+ muted: member.muted,+ })),+ }+ : {}),+ saved_messages: savedMessages.map((msg) => ({+ channel_id: msg.channelId.toString(),+ message_id: msg.messageId.toString(),+ saved_at: msg.savedAt.toISOString(),+ })),+ private_channels: privateChannels.map((channel) => ({+ channel_id: channel.id.toString(),+ type: channel.type,+ name: channel.name,+ icon_hash: channel.iconHash,+ owner_id: channel.ownerId?.toString() ?? null,+ recipient_ids: Array.from(channel.recipientIds).map((id) => id.toString()),+ last_message_id: channel.lastMessageId?.toString() ?? null,+ })),+ favorite_memes: favoriteMemes.map((meme) => ({+ meme_id: meme.id.toString(),+ name: meme.name,+ alt_text: meme.altText,+ tags: meme.tags,+ filename: meme.filename,+ content_type: meme.contentType,+ size: meme.size.toString(),+ width: meme.width,+ height: meme.height,+ duration: meme.duration,+ })),+ push_subscriptions: pushSubscriptions.map((sub) => ({+ subscription_id: sub.subscriptionId,+ platform: sub.platform,+ app_id: sub.appId,+ provider_environment: sub.providerEnvironment,+ endpoint: sub.endpoint,+ user_agent: sub.userAgent,+ })),+ webauthn_credentials: webAuthnCredentials.map((cred) => ({+ credential_id: cred.credentialId,+ name: cred.name,+ transports: cred.transports ? Array.from(cred.transports) : [],+ created_at: cred.createdAt.toISOString(),+ last_used_at: cred.lastUsedAt?.toISOString() ?? null,+ })),+ mfa_backup_codes: {+ total_count: mfaBackupCodes.length,+ consumed_count: mfaBackupCodes.filter((code) => code.consumed).length,+ remaining_count: mfaBackupCodes.filter((code) => !code.consumed).length,+ },+ ...(recoveryKitCreatedAt ? {recovery_kit: {created_at: recoveryKitCreatedAt.toISOString()}} : {}),+ gift_codes_created: createdGiftCodes.map((gift) => ({+ code: gift.code,+ duration_months: gift.durationMonths,+ duration_type: gift.durationType,+ duration_quantity: gift.durationQuantity,+ created_at: gift.createdAt.toISOString(),+ redeemed_by_user_id: gift.redeemedByUserId?.toString() ?? null,+ redeemed_at: gift.redeemedAt?.toISOString() ?? null,+ stripe_payment_intent_id: gift.stripePaymentIntentId,+ })),+ payments: payments.map(mapPayment),+ store_purchases: storePurchases.map(mapStorePurchaseToResponse),+ operator_pass: operatorPass+ ? {+ granted_at: operatorPass.granted_at.toISOString(),+ source: operatorPass.source,+ checkout_session_id: operatorPass.checkout_session_id ?? null,+ pass_count: operatorPass.pass_count ?? 1,+ payment_intent_id: operatorPass.payment_intent_id ?? null,+ amount_minor: operatorPass.amount_minor ?? null,+ currency: operatorPass.currency ?? null,+ paused: operatorPass.paused_at != null,+ terms_version: operatorPass.terms_version ?? null,+ terms_accepted_at: operatorPass.terms_accepted_at?.toISOString() ?? null,+ waiver_consent_at: operatorPass.waiver_consent_at?.toISOString() ?? null,+ waiver_consent_version: operatorPass.waiver_consent_version ?? null,+ waiver_consent_ip: operatorPass.waiver_consent_ip ?? null,+ waiver_consent_user_agent: operatorPass.waiver_consent_user_agent ?? null,+ }+ : null,+ operator_pass_wall: operatorPassWall+ ? {+ name: operatorPassWall.name ?? null,+ opted_out: operatorPassWall.opted_out,+ keep_as_is: operatorPassWall.keep_as_is,+ release_list_opt_in: operatorPassWall.release_list_opt_in ?? false,+ hidden: operatorPassWall.hidden,+ live_name: operatorPassWall.live_name ?? null,+ business: operatorPassWall.live_business ?? false,+ pending_name: operatorPassWall.pending_name ?? null,+ declined_name: operatorPassWall.declined_name ?? null,+ decline_reason: operatorPassWall.decline_reason ?? null,+ decline_note: operatorPassWall.decline_note ?? null,+ listed_at: operatorPassWall.listed_at?.toISOString() ?? null,+ delivered_at: operatorPassWall.delivered_at?.toISOString() ?? null,+ release_published_year: operatorPassWall.release_published_year ?? null,+ release_erased_at: operatorPassWall.release_erased_at?.toISOString() ?? null,+ }+ : null,+ operator_pass_wall_history: operatorPassWallEvents.map((event) => ({+ occurred_at: event.occurred_at.toISOString(),+ kind: event.kind,+ name: event.name ?? null,+ reason: event.reason ?? null,+ cause: event.cause ?? null,+ purchase_id: operatorPassPurchases.some((purchase) => purchase.checkout_session_id === event.purchase_id)+ ? (event.purchase_id ?? null)+ : null,+ release_year: event.release_year ?? null,+ release_tag: event.release_tag ?? null,+ pass_count: event.pass_count ?? null,+ })),+ operator_pass_purchases: operatorPassPurchases.map((purchase) => ({+ checkout_session_id: purchase.checkout_session_id,+ quantity: purchase.quantity ?? 1,+ status: purchase.status,+ phase: purchase.phase,+ amount_minor: purchase.amount_minor,+ currency: purchase.currency,+ quote_honored: purchase.quote_honored,+ payment_intent_id: purchase.payment_intent_id ?? null,+ invoice_id: purchase.invoice_id ?? null,+ subscription_id: purchase.subscription_id ?? null,+ created_at: purchase.created_at.toISOString(),+ completed_at: purchase.completed_at?.toISOString() ?? null,+ refunded_at: purchase.refunded_at?.toISOString() ?? null,+ terms_version: purchase.operator_pass_terms_version ?? null,+ purchased_at: purchase.purchased_at?.toISOString() ?? null,+ delivered_at: purchase.perks_granted_at?.toISOString() ?? null,+ withdrawal_ends_at: purchase.withdrawal_ends_at?.toISOString() ?? null,+ withdrawn_at: purchase.withdrawn_at?.toISOString() ?? null,+ withdrawal_name: purchase.withdrawal_name ?? null,+ waiver_consent_at: purchase.waiver_consent_at?.toISOString() ?? null,+ waiver_consent_version: purchase.waiver_consent_version ?? null,+ waiver_consent_text: purchase.waiver_consent_text ?? null,+ waiver_consent_ip: purchase.waiver_consent_ip ?? null,+ waiver_consent_user_agent: purchase.waiver_consent_user_agent ?? null,+ wall_name: purchase.wall_name ?? null,+ wall_opt_out: purchase.wall_opt_out ?? null,+ withdrawal_country: purchase.withdrawal_country ?? null,+ withdrawal_days: purchase.withdrawal_days ?? null,+ withdrawal_rule_version: purchase.withdrawal_rule_version ?? null,+ stripe_terms_consent: purchase.stripe_terms_consent ?? null,+ })),+ operator_passes: operatorPassUnits.map(({unit, events}) => mapOperatorPassUnit(unit, events, userId)),+ operator_pass_offers: operatorPassOffers.map((offer) => ({+ id: offer.offer_id,+ pass_id: offer.sender_id === userId ? offer.pass_id : null,+ status: offer.status,+ sender_id: offer.sender_id.toString(),+ recipient_id: offer.recipient_id.toString(),+ created_at: offer.created_at.toISOString(),+ accepted_at: offer.accepted_at?.toISOString() ?? null,+ arrives_at: offer.completes_at?.toISOString() ?? null,+ ended_at: offer.ended_at?.toISOString() ?? null,+ wall_name: offer.recipient_id === userId ? (offer.wall_name ?? null) : null,+ terms_version: offer.terms_version ?? null,+ })),+ operator_pass_waitlist: operatorPassWaitlistEntry+ ? {+ status: operatorPassWaitlistEntry.status,+ locale: operatorPassWaitlistEntry.locale ?? null,+ country_code: operatorPassWaitlistEntry.country_code ?? null,+ consent_ip: operatorPassWaitlistEntry.consent_ip ?? null,+ price_feedback: operatorPassWaitlistEntry.price_feedback ?? null,+ currency: operatorPassWaitlistEntry.currency ?? null,+ followup_picks: [...(operatorPassWaitlistEntry.followup_picks ?? [])].sort(),+ followup_about: [...(operatorPassWaitlistEntry.followup_about ?? [])].sort(),+ followup_other_text: operatorPassWaitlistEntry.followup_other_text ?? null,+ followup_answered_at: operatorPassWaitlistEntry.followup_answered_at?.toISOString() ?? null,+ created_at: operatorPassWaitlistEntry.created_at.toISOString(),+ updated_at: operatorPassWaitlistEntry.updated_at.toISOString(),+ confirmed_at: operatorPassWaitlistEntry.confirmed_at?.toISOString() ?? null,+ unsubscribed_at: operatorPassWaitlistEntry.unsubscribed_at?.toISOString() ?? null,+ sale_email_sent_at: operatorPassWaitlistEntry.sale_email_sent_at?.toISOString() ?? null,+ }+ : null,+ oauth_applications: oauthClients.map(mapOAuthApplication),+ connections: connections.map((connection) => ({+ id: connection.connection_id,+ type: connection.connection_type,+ identifier: connection.identifier,+ name: connection.name,+ verified: connection.verified,+ visibility_flags: connection.visibility_flags,+ sort_order: connection.sort_order,+ created_at: connection.created_at.toISOString(),+ verified_at: connection.verified_at?.toISOString() ?? null,+ last_verified_at: connection.last_verified_at?.toISOString() ?? null,+ })),+ pinned_dms: pinnedDms.map((pin) => ({+ channel_id: pin.channel_id.toString(),+ sort_order: pin.sort_order,+ })),+ ...mapSecurityData({authorizedIps, activityData}),+ };+}