A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
test(api/worker): verify data exports through stored archives
Hampus2bb7d566893d
diff --git a/fluxer_api/src/api/worker/tasks/HarvestUserData.test.ts b/fluxer_api/src/api/worker/tasks/HarvestUserData.test.tsindex 150b0e345..e62e885ad 100644--- a/fluxer_api/src/api/worker/tasks/HarvestUserData.test.ts+++ b/fluxer_api/src/api/worker/tasks/HarvestUserData.test.ts@@ -3,171 +3,392 @@import {type ChannelID,createChannelID,+ createEmojiID,createGuildID,createMessageID,createUserID,- type GuildID,- type MessageID,- type UserID,} from '@app/api/BrandedTypes';+import {Config} from '@app/api/Config';+import {upsertOne} from '@app/api/database/CassandraQueryExecution';+import type {MessageRow} from '@app/api/database/types/MessageTypes';+import type {StorePurchaseRow} from '@app/api/database/types/StoreBillingTypes';+import type {ThreadMemberRow} from '@app/api/database/types/ThreadTypes';import {EMPTY_USER_ROW} from '@app/api/database/types/UserTypes';-import {ThreadMember} from '@app/api/models/ThreadMember';-import {User} from '@app/api/models/User';-import {mapStorePurchaseToResponse} from '@app/api/store_billing/StoreBillingMappers';+import {clearChannelThreadsTaintCacheForTesting, insertGuildThreadMarker} from '@app/api/experiment/ChannelThreadsGate';+import {getSnowflakeService} from '@app/api/middleware/ServiceRegistry';import {buildStorePurchaseRow} from '@app/api/store_billing/tests/StoreBillingTestUtils';-import {buildUserDataJson, collectThreadMemberships, harvestMessages} from '@app/api/worker/tasks/HarvestUserData';-import {afterEach, describe, expect, it, vi} from 'vitest';--const tainted = vi.hoisted(() => new Set<string>());--vi.mock('@app/api/experiment/ChannelThreadsGate', async (importOriginal) => ({- ...(await importOriginal<typeof import('@app/api/experiment/ChannelThreadsGate')>()),- isTainted: async (guildId: bigint) => tainted.has(guildId.toString()),-}));+import {+ GuildMembersByUserId,+ Messages,+ MessagesByAuthorV2,+ ThreadMembers,+ ThreadMembersByUser,+ UserRecoveryKits,+} from '@app/api/Tables';+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';+import {HarvestedMessagesSchema, harvestUserArchive} from '@app/api/test/UserArchiveTestUtils';+import {clearWorkerDependencies, setWorkerDependencies} from '@app/api/worker/WorkerContext';+import {initializeWorkerDependencies, type WorkerDependencies} from '@app/api/worker/WorkerDependencies';+import {AccountIdentityModes} from '@fluxer/constants/src/AccountIdentityConstants';+import {makeBucket} from '@fluxer/snowflake/src/SnowflakeBuckets';+import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it} from 'vitest';+import {z} from 'zod';const AUTHOR = createUserID(1000000000000000000n);+let harness: ApiTestHarness;+let deps: WorkerDependencies;+let selfHosted: boolean;++beforeAll(async () => {+ harness = await createApiTestHarness();+});+beforeEach(async () => {+ await harness.reset();+ clearChannelThreadsTaintCacheForTesting();+ selfHosted = Config.instance.selfHosted;+ deps = await initializeWorkerDependencies(getSnowflakeService());+ setWorkerDependencies(deps);+ await deps.userRepository.create({+ ...EMPTY_USER_ROW,+ user_id: AUTHOR,+ username: 'exporter',+ discriminator: 1,+ email: '[email protected]',+ });+});+afterEach(() => {+ Config.instance.selfHosted = selfHosted;+ clearChannelThreadsTaintCacheForTesting();+ clearWorkerDependencies();+});+afterAll(async () => {+ await harness.shutdown();+});-function makeRepository(refs: Array<{channelId: ChannelID; messageId: MessageID}>, missing = new Set<string>()) {- let pages = 0;+function messageRow(channelId: ChannelID, id: bigint): MessageRow {+ const messageId = createMessageID(id);return {- pages: () => pages,- listMessagesByAuthor: async (_userId: typeof AUTHOR, limit: number, lastMessageId?: MessageID) => {- pages++;- const start = lastMessageId ? refs.findIndex((r) => r.messageId === lastMessageId) + 1 : 0;- return refs.slice(start, start + limit);- },- getMessage: async (_channelId: ChannelID, messageId: MessageID) =>- missing.has(messageId.toString()) ? null : {content: `body ${messageId.toString()}`, attachments: undefined},+ channel_id: channelId,+ bucket: makeBucket(messageId),+ message_id: messageId,+ author_id: AUTHOR,+ type: 0,+ webhook_id: null,+ webhook_name: null,+ webhook_avatar_hash: null,+ content: `body ${id}`,+ edited_timestamp: null,+ pinned_timestamp: null,+ flags: 0,+ mention_everyone: false,+ mention_users: null,+ mention_roles: null,+ mention_channels: null,+ attachments: null,+ embeds: null,+ sticker_items: null,+ message_reference: null,+ message_snapshots: null,+ call: null,+ has_reaction: false,+ version: 1,};}-function refsAcross(channelCount: number, perChannel: number) {- const out: Array<{channelId: ChannelID; messageId: MessageID}> = [];- let id = 1500000000000000000n;- for (let c = 0; c < channelCount; c++) {- const channelId = createChannelID(2000000000000000000n + BigInt(c));- for (let m = 0; m < perChannel; m++) {- out.push({channelId, messageId: createMessageID(id)});- id += 1n;- }+async function seedMessages(channelId: ChannelID, count: number, firstId = 1500000000000000000n) {+ for (let index = 0; index < count; index++) {+ const row = messageRow(channelId, firstId + BigInt(index));+ await upsertOne(Messages.upsertAll(row));+ await upsertOne(+ MessagesByAuthorV2.upsertAll({author_id: AUTHOR, channel_id: channelId, message_id: row.message_id}),+ );}- return out;}-describe('harvestMessages', () => {- it('reads past a single page instead of stopping at one query', async () => {- const refs = refsAcross(1, 2500);- const repo = makeRepository(refs);- const result = await harvestMessages(repo, AUTHOR, Date.now(), null);- expect(result.totalMessages).toBe(2500);- expect(repo.pages()).toBeGreaterThan(1);+describe('harvested messages', () => {+ it('archives every message beyond the first page', async () => {+ await seedMessages(createChannelID(2000000000000000000n), 2500);+ const archive = await harvestUserArchive(harness, deps, AUTHOR);+ const messages = HarvestedMessagesSchema.parse(archive.readJson('channels/2000000000000000000/messages.json'));+ expect(messages).toHaveLength(2500);+ expect(new Set(messages.map((message) => message.id)).size).toBe(2500);+ expect(messages).toContainEqual({id: '1500000000000000000', content: 'body 1500000000000000000'});+ expect(messages).toContainEqual({id: '1500000000000002499', content: 'body 1500000000000002499'});});- it('groups every message under its own channel', async () => {- const repo = makeRepository(refsAcross(3, 4));- const result = await harvestMessages(repo, AUTHOR, Date.now(), null);- expect(result.channelMessagesMap.size).toBe(3);- for (const messages of result.channelMessagesMap.values()) {- expect(messages).toHaveLength(4);+ it('groups messages into the archive file for their channel', async () => {+ await seedMessages(createChannelID(2000000000000000000n), 4);+ await seedMessages(createChannelID(2000000000000000001n), 4, 1500000000000000004n);+ await seedMessages(createChannelID(2000000000000000002n), 4, 1500000000000000008n);+ const archive = await harvestUserArchive(harness, deps, AUTHOR);+ for (const name of [+ 'channels/2000000000000000000/messages.json',+ 'channels/2000000000000000001/messages.json',+ 'channels/2000000000000000002/messages.json',+ ]) {+ expect(HarvestedMessagesSchema.parse(archive.readJson(name))).toHaveLength(4);}- expect(result.totalMessages).toBe(12);});- it('leaves out a message the repository cannot return', async () => {- const refs = refsAcross(1, 5);- const repo = makeRepository(refs, new Set([refs[2].messageId.toString()]));- const result = await harvestMessages(repo, AUTHOR, Date.now(), null);- expect(result.totalMessages).toBe(4);+ it('leaves out stale author references to missing messages', async () => {+ const channelId = createChannelID(2000000000000000000n);+ await seedMessages(channelId, 2);+ await upsertOne(+ MessagesByAuthorV2.upsertAll({+ author_id: AUTHOR,+ channel_id: channelId,+ message_id: createMessageID(1500000000000000002n),+ }),+ );+ const archive = await harvestUserArchive(harness, deps, AUTHOR);+ expect(HarvestedMessagesSchema.parse(archive.readJson('channels/2000000000000000000/messages.json'))).toEqual([+ {id: '1500000000000000001', content: 'body 1500000000000000001'},+ {id: '1500000000000000000', content: 'body 1500000000000000000'},+ ]);});- it('returns an empty map for an account with no messages', async () => {- const repo = makeRepository([]);- const result = await harvestMessages(repo, AUTHOR, Date.now(), null);- expect(result.totalMessages).toBe(0);- expect(result.channelMessagesMap.size).toBe(0);+ it('creates no channel files for an account without messages', async () => {+ const archive = await harvestUserArchive(harness, deps, AUTHOR);+ expect(Object.keys(archive.entries).sort()).toEqual([+ 'account/security.json',+ 'integrations/oauth.json',+ 'payments/payment_history.json',+ 'user.json',+ ]);});});-function userDataParams(overrides: Partial<Parameters<typeof buildUserDataJson>[0]> = {}) {+function threadMemberRow(overrides: Partial<ThreadMemberRow> = {}): ThreadMemberRow {return {- user: new User({...EMPTY_USER_ROW, user_id: AUTHOR, username: 'exporter', discriminator: 1}),- userId: AUTHOR,- productName: 'Fluxer',- authSessions: [],- relationships: [],- userNotes: new Map(),- userSettings: null,- guildMemberships: [],- guildSettings: [],- savedMessages: [],- privateChannels: [],- favoriteMemes: [],- pushSubscriptions: [],- webAuthnCredentials: [],- mfaBackupCodes: [],- recoveryKitCreatedAt: null,- createdGiftCodes: [],- payments: [],- storePurchases: [],- threadMemberships: [],- oauthClients: [],- connections: [],- pinnedDms: [],- authorizedIps: [],- activityData: {last_active_at: null, last_active_ip: null},+ thread_id: createChannelID(1400000000000000010n),+ user_id: AUTHOR,+ guild_id: createGuildID(1400000000000000000n),+ parent_id: createChannelID(1400000000000000001n),+ join_timestamp: new Date('2026-09-27T00:00:00.000Z'),+ flags: 1,+ muted: false,+ mute_config: null,...overrides,};}-describe('buildUserDataJson', () => {- it('exports the store purchases in their public shape', () => {- const purchase = buildStorePurchaseRow({user_id: AUTHOR});- const data = buildUserDataJson(userDataParams({storePurchases: [purchase]}));- expect(data.store_purchases).toEqual([mapStorePurchaseToResponse(purchase)]);+describe('harvested thread memberships', () => {+ it('includes joined threads from marked guilds and omits stale rows from unmarked guilds', async () => {+ const marked = threadMemberRow();+ const unmarked = threadMemberRow({+ guild_id: createGuildID(1400000000000000002n),+ thread_id: createChannelID(1400000000000000020n),+ });+ for (const member of [marked, unmarked]) {+ await upsertOne(GuildMembersByUserId.upsertAll({user_id: AUTHOR, guild_id: member.guild_id}));+ await upsertOne(ThreadMembers.upsertAll(member));+ await upsertOne(+ ThreadMembersByUser.upsertAll({+ user_id: AUTHOR,+ guild_id: member.guild_id,+ parent_id: member.parent_id,+ is_private: false,+ thread_id: member.thread_id,+ }),+ );+ }+ await insertGuildThreadMarker(marked.guild_id, null);+ const archive = await harvestUserArchive(harness, deps, AUTHOR);+ expect(archive.readJson('user.json')).toMatchObject({+ thread_memberships: [+ {+ thread_id: '1400000000000000010',+ guild_id: '1400000000000000000',+ parent_id: '1400000000000000001',+ join_timestamp: '2026-09-27T00:00:00.000Z',+ flags: 1,+ muted: false,+ },+ ],+ });+ });+});++interface SeedData {+ storePurchases?: Array<StorePurchaseRow>;+ recoveryKitCreatedAt?: Date;+}++const UserDataSchema = z.looseObject({+ operator_passes: z.array(z.looseObject({events: z.array(z.looseObject({kind: z.string()}))})),+});++async function exportUserData(data: SeedData = {}) {+ for (const purchase of data.storePurchases ?? []) await getStoreBillingRepository().insertPurchase(purchase);+ if (data.recoveryKitCreatedAt) {+ Config.instance.selfHosted = true;+ await deps.instanceConfigRepository.setAccountIdentityMode(AccountIdentityModes.USERNAME, 'setup');+ await upsertOne(+ UserRecoveryKits.upsertAll({+ user_id: userId,+ secret_hash: 'recovery-secret',+ created_at: data.recoveryKitCreatedAt,+ }),+ );+ }+ const archive = await harvestUserArchive(harness, deps, userId);+ return UserDataSchema.parse(archive.readJson('user.json'));+}++describe('harvested user data', () => {+ it('exports the store purchases in their public shape', async () => {+ const purchase = buildStorePurchaseRow({+ user_id: AUTHOR,+ id: 1234n,+ created_at: new Date('2026-11-01T00:00:00Z'),+ expires_at: new Date('2026-12-01T00:00:00Z'),+ });+ const data = await exportUserData({storePurchases: [purchase]});+ expect(data.store_purchases).toEqual([+ {+ id: '1234',+ provider: 'app_store',+ kind: 'subscription',+ slot: 'monthly',+ product_id: 'com.fluxer.plutonium.monthly',+ environment: 'production',+ state: 'active',+ entitled: true,+ expires_at: '2026-12-01T00:00:00.000Z',+ entitled_until: '2026-12-01T00:00:00.000Z',+ will_renew: true,+ gift_code: null,+ created_at: '2026-11-01T00:00:00.000Z',+ },+ ]);});- it('exports when the recovery kit was created and nothing else about it', () => {+ it('exports when the recovery kit was created and nothing else about it', async () => {const createdAt = new Date('2026-10-01T12:00:00.000Z');- const data = buildUserDataJson(userDataParams({recoveryKitCreatedAt: createdAt}));+ const data = await exportUserData({recoveryKitCreatedAt: createdAt});expect(data.recovery_kit).toEqual({created_at: '2026-10-01T12:00:00.000Z'});});- it('leaves the recovery kit out when the account has none', () => {- const data = buildUserDataJson(userDataParams());+ it('leaves the recovery kit out when the account has none', async () => {+ const data = await exportUserData({});expect('recovery_kit' in data).toBe(false);});});-describe('collectThreadMemberships', () => {- const guildId = createGuildID(1400000000000000000n);- const threadId = createChannelID(1400000000000000010n);- const member = new ThreadMember({- thread_id: threadId,- user_id: AUTHOR,- guild_id: guildId,- parent_id: createChannelID(1400000000000000001n),- join_timestamp: new Date('2026-09-27T00:00:00.000Z'),- flags: 1,- muted: false,- mute_config: null,- });- let reads = 0;- const threads = {- listJoinedThreadIds: async (_userId: UserID, id: GuildID) => {- reads++;- return id === guildId ? [threadId] : [];+describe('harvested user settings', () => {+ it.each([+ {name: 'no custom status', customStatus: null, expected: null},+ {+ name: 'complete custom status',+ customStatus: {+ text: 'Away for lunch',+ emoji_id: createEmojiID(4000000000000000000n),+ emoji_name: 'lunch',+ emoji_animated: true,+ expires_at: new Date('2027-01-02T03:04:05.000Z'),+ },+ expected: {+ text: 'Away for lunch',+ emoji_id: '4000000000000000000',+ emoji_name: 'lunch',+ emoji_animated: true,+ expires_at: '2027-01-02T03:04:05.000Z',+ },},- getMember: async () => member,- };- afterEach(() => {- reads = 0;- tainted.clear();- });-- it('collects the joined thread memberships of every tainted guild and skips the rest', async () => {- tainted.add(guildId.toString());- const collected = await collectThreadMemberships(threads, AUTHOR, [guildId, createGuildID(1400000000000000002n)]);- expect(collected).toEqual([member]);- expect(reads).toBe(1);+ {+ name: 'nullable custom status fields',+ customStatus: {text: null, emoji_id: null, emoji_name: null, emoji_animated: false, expires_at: null},+ expected: {text: null, emoji_id: null, emoji_name: null, emoji_animated: false, expires_at: null},+ },+ ])('writes the complete settings document with $name', async ({customStatus, expected}) => {+ await deps.userRepository.upsertSettings({+ user_id: AUTHOR,+ locale: 'sv-SE',+ theme: 'dark',+ status: 'dnd',+ status_resets_at: null,+ status_resets_to: null,+ custom_status: customStatus,+ developer_mode: true,+ message_display_compact: true,+ animate_emoji: false,+ animate_stickers: 2,+ gif_auto_play: true,+ render_embeds: true,+ render_reactions: false,+ render_spoilers: 1,+ inline_attachment_media: true,+ inline_embed_media: false,+ explicit_content_filter: 2,+ friend_source_flags: 3,+ incoming_call_flags: 0,+ group_dm_add_permission_flags: 0,+ default_guilds_restricted: true,+ bot_default_guilds_restricted: false,+ restricted_guilds: new Set([createGuildID(2000000000000000000n)]),+ bot_restricted_guilds: new Set([createGuildID(3000000000000000000n)]),+ guild_positions: [createGuildID(3000000000000000000n), createGuildID(2000000000000000000n)],+ guild_folders: [+ {+ folder_id: 4,+ name: 'Notes',+ color: 42,+ flags: 1,+ icon: 'folder',+ guild_ids: [createGuildID(3000000000000000000n)],+ },+ ],+ afk_timeout: 300,+ time_format: 1,+ trusted_domains: new Set(['example.com']),+ default_hide_muted_channels: false,+ sensitive_content_friend_dm_filter: 0,+ sensitive_content_non_friend_dm_filter: 0,+ sensitive_content_guild_filter: 0,+ suppress_unprivileged_self_mentions: true,+ suppress_unprivileged_self_mentions_bypass_user_ids: new Set([createUserID(6000000000000000000n)]),+ staff_dm_access_user_ids: new Set([createUserID(7000000000000000000n)]),+ synced_preferences: null,+ profile_privacy: 2,+ default_share_voice_activity: false,+ privacy_setup_version: 3,+ privacy_setup_completed_at: new Date('2026-10-01T12:00:00.000Z'),+ version: 1,+ });+ const archive = await harvestUserArchive(harness, deps, AUTHOR);+ const {user_settings} = z.object({user_settings: z.unknown()}).parse(archive.readJson('user.json'));+ expect(user_settings).toEqual({+ locale: 'sv-SE',+ theme: 'dark',+ status: 'dnd',+ custom_status: expected,+ developer_mode: true,+ message_display_compact: true,+ animate_emoji: false,+ animate_stickers: 2,+ gif_auto_play: true,+ render_embeds: true,+ render_reactions: false,+ render_spoilers: 1,+ inline_attachment_media: true,+ inline_embed_media: false,+ explicit_content_filter: 2,+ friend_source_flags: 3,+ default_guilds_restricted: true,+ bot_default_guilds_restricted: false,+ restricted_guilds: ['2000000000000000000'],+ bot_restricted_guilds: ['3000000000000000000'],+ guild_positions: ['3000000000000000000', '2000000000000000000'],+ guild_folders: [{id: 4, name: 'Notes', color: 42, flags: 1, icon: 'folder', guild_ids: ['3000000000000000000']}],+ afk_timeout: 300,+ time_format: 1,+ suppress_unprivileged_self_mentions: true,+ suppress_unprivileged_self_mentions_bypass_user_ids: ['6000000000000000000'],+ staff_dm_access_user_ids: ['7000000000000000000'],+ synced_preferences: {},+ profile_privacy: 2,+ privacy_setup_version: 3,+ privacy_setup_completed_at: '2026-10-01T12:00:00.000Z',+ });});});