A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
test(api): read data export and sweep tests from stored archives
Hampus13429ebfb252
diff --git a/fluxer_api/src/api/test/UserArchiveTestUtils.ts b/fluxer_api/src/api/test/UserArchiveTestUtils.tsnew file mode 100644index 000000000..c193095fb--- /dev/null+++ b/fluxer_api/src/api/test/UserArchiveTestUtils.ts@@ -0,0 +1,55 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import assert from 'node:assert/strict';+import type {UserID} from '@app/api/BrandedTypes';+import {Config} from '@app/api/Config';+import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';+import {createTestWorkerHelpers} from '@app/api/test/SyncTaskWorkerService';+import {UserHarvest} from '@app/api/user/UserHarvestModel';+import harvestUserData from '@app/api/worker/tasks/HarvestUserData';+import type {WorkerDependencies} from '@app/api/worker/WorkerDependencies';+import {strFromU8, unzipSync} from 'fflate';+import {z} from 'zod';++export async function harvestUserArchive(harness: ApiTestHarness, deps: WorkerDependencies, userId: UserID) {+ const harvestId = await deps.snowflakeService.generate();+ await deps.userHarvestRepository.create(+ new UserHarvest({+ user_id: userId,+ harvest_id: harvestId,+ requested_at: new Date(),+ started_at: null,+ completed_at: null,+ failed_at: null,+ storage_key: null,+ file_size: null,+ progress_percent: 0,+ progress_step: null,+ error_message: null,+ download_url_expires_at: null,+ }),+ );+ await harvestUserData(+ {userId: userId.toString(), harvestId: harvestId.toString()},+ createTestWorkerHelpers({attempt: {isLastAttempt: true}}),+ );+ const completed = await deps.userHarvestRepository.findByUserAndHarvestId(userId, harvestId);+ assert.equal(completed?.getStatus(), 'completed');+ assert(completed?.storageKey);+ return readStoredArchive(harness, completed.storageKey);+}++export async function readStoredArchive(harness: ApiTestHarness, storageKey: string) {+ const zip = await harness.storageService.readObject(Config.s3.buckets.harvests, storageKey);+ const entries = unzipSync(zip);+ return {+ entries,+ readJson(name: string): unknown {+ const bytes = entries[name];+ assert(bytes, `Archive is missing ${name}`);+ return JSON.parse(strFromU8(bytes));+ },+ };+}++export const HarvestedMessagesSchema = z.array(z.object({id: z.string(), content: z.string()}));diff --git a/fluxer_api/src/api/user/tests/HarvestTestUtils.ts b/fluxer_api/src/api/user/tests/HarvestTestUtils.tsindex 8cb334ecf..280dc07ab 100644--- a/fluxer_api/src/api/user/tests/HarvestTestUtils.ts+++ b/fluxer_api/src/api/user/tests/HarvestTestUtils.ts@@ -6,7 +6,6 @@ import {createBuilder} from '@app/api/test/TestRequestBuilder';import type {UserHarvest} from '@app/api/user/UserHarvestModel';import {UserHarvestRepository} from '@app/api/user/UserHarvestRepository';import type {HarvestDownloadUrlResponse} from '@fluxer/schema/src/domains/user/UserHarvestSchemas';-import {expect} from 'vitest';interface HarvestRequestResponse {harvest_id: string;@@ -26,40 +25,12 @@ export async function fetchHarvestDownload(.execute();}-export async function expectHarvestDownloadFailsWithError(- harness: ApiTestHarness,- token: string,- harvestId: string,- expectedCode: string,-): Promise<void> {- const {json} = await createBuilder<Record<string, unknown>>(harness, token)- .get(`/users/@me/harvest/${harvestId}/download`)- .expect(400)- .executeWithResponse();- const errorResponse = json as {- code: string;- message: string;- };- expect(errorResponse.code).toBe(expectedCode);-}-export async function markHarvestCompleted(userId: string, harvestId: string, expiresAt: Date): Promise<void> {const harvestRepository = new UserHarvestRepository();const harvest = await claimHarvest(harvestRepository, userId, harvestId);await harvestRepository.markAsCompleted(harvest, `test/${harvestId}.zip`, 1024n, expiresAt);}-export async function markHarvestFailed(userId: string, harvestId: string, errorMessage: string): Promise<void> {- const harvestRepository = new UserHarvestRepository();- const harvest = await claimHarvest(harvestRepository, userId, harvestId);- await harvestRepository.markAsFailed(harvest, errorMessage);-}--export async function markHarvestStarted(userId: string, harvestId: string): Promise<void> {- const harvestRepository = new UserHarvestRepository();- await claimHarvest(harvestRepository, userId, harvestId);-}-async function claimHarvest(repository: UserHarvestRepository,userId: string,@@ -69,8 +40,3 @@ async function claimHarvest(if (!harvest) throw new Error(`Harvest ${harvestId} for user ${userId} not found`);return repository.markAsStarted(harvest);}--export async function findHarvest(userId: string, harvestId: string): Promise<UserHarvest | null> {- const harvestRepository = new UserHarvestRepository();- return harvestRepository.findByUserAndHarvestId(createUserID(BigInt(userId)), BigInt(harvestId));-}diff --git a/fluxer_api/src/api/worker/tests/ThreadMaintenanceSweeps.test.ts b/fluxer_api/src/api/worker/tests/ThreadMaintenanceSweeps.test.tsindex 7b177472c..ff21f97ef 100644--- a/fluxer_api/src/api/worker/tests/ThreadMaintenanceSweeps.test.ts+++ b/fluxer_api/src/api/worker/tests/ThreadMaintenanceSweeps.test.ts@@ -1,6 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later-import fs from 'node:fs';+import assert from 'node:assert/strict';import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';import {createChannelID, createGuildID, createUserID} from '@app/api/BrandedTypes';import {acceptInvite, createChannel, createChannelInvite, createGuild} from '@app/api/channel/tests/ChannelTestUtils';@@ -8,16 +8,15 @@ import {threadsRequest} from '@app/api/channel/tests/ThreadTestUtils';import {deleteOneOrMany, upsertOne} from '@app/api/database/CassandraQueryExecution';import {clearChannelThreadsTaintCacheForTesting, isTainted} from '@app/api/experiment/ChannelThreadsGate';import {ensureSessionStarted} from '@app/api/message/tests/MessageTestUtils';-import {getSnowflakeService} from '@app/api/middleware/ServiceRegistry';+import {getSnowflakeService, setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';import {getChannelRepository} from '@app/api/middleware/ServiceSingletons';import {GuildThreadState} from '@app/api/Tables';import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';-import {NoopWorkerService} from '@app/api/test/NoopWorkerService';-import {createTestWorkerHelpers} from '@app/api/test/SyncTaskWorkerService';+import {SyncTaskWorkerService} from '@app/api/test/SyncTaskWorkerService';import {createBuilder} from '@app/api/test/TestRequestBuilder';+import {HarvestedMessagesSchema, harvestUserArchive, readStoredArchive} from '@app/api/test/UserArchiveTestUtils';import deleteUserMessagesInGuildByTime from '@app/api/worker/tasks/DeleteUserMessagesInGuildByTime';import harvestGuildData from '@app/api/worker/tasks/HarvestGuildData';-import {harvestMessages} from '@app/api/worker/tasks/HarvestUserData';import {clearWorkerDependencies, setWorkerDependencies} from '@app/api/worker/WorkerContext';import {initializeWorkerDependencies, type WorkerDependencies} from '@app/api/worker/WorkerDependencies';import {AdminACLs} from '@fluxer/constants/src/AdminACLs';@@ -25,7 +24,7 @@ import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants';import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas';import type {ThreadChannelResponse} from '@fluxer/schema/src/domains/channel/ThreadRequestSchemas';import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas';-import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';+import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';interface Setup {owner: TestAccount;@@ -39,7 +38,7 @@ interface Setup {describe('thread maintenance sweeps', () => {let harness: ApiTestHarness;let deps: WorkerDependencies;- let jobs: Array<{task: string; payload: Record<string, unknown>}>;+ let worker: SyncTaskWorkerService;beforeAll(async () => {harness = await createApiTestHarness();@@ -48,25 +47,16 @@ describe('thread maintenance sweeps', () => {beforeEach(async () => {await harness.reset();clearChannelThreadsTaintCacheForTesting();+ worker = new SyncTaskWorkerService({deleteUserMessagesInGuildByTime, harvestGuildData}, {deferred: true});+ setInjectedWorkerService(worker);deps = await initializeWorkerDependencies(getSnowflakeService());setWorkerDependencies(deps);- jobs = [];- const addJob = async (task: string, payload: Record<string, unknown>) => {- jobs.push({task, payload});- if (task === 'deleteUserMessagesInGuildByTime')- await deleteUserMessagesInGuildByTime(payload, createTestWorkerHelpers({attempt: {isLastAttempt: true}}));- return 0n;- };- vi.spyOn(NoopWorkerService.prototype, 'addJob').mockImplementation(addJob as () => Promise<bigint>);- });-- afterEach(() => {- vi.restoreAllMocks();});afterAll(async () => {clearWorkerDependencies();clearChannelThreadsTaintCacheForTesting();+ setInjectedWorkerService(undefined);await harness.shutdown();});@@ -109,6 +99,7 @@ describe('thread maintenance sweeps', () => {.body({delete_message_seconds: 3600}).expect(204).execute();+ expect((await worker.drain()).failed).toBe(0);}it('removes thread messages inside the ban delete window and decrements the count', async () => {@@ -143,7 +134,8 @@ describe('thread maintenance sweeps', () => {await deleteOneOrMany(GuildThreadState.deleteByPk({guild_id: guildId}));clearChannelThreadsTaintCacheForTesting();expect(await isTainted(guildId)).toBe(false);- await upsertOne(GuildThreadState.upsertAll(marker!));+ assert(marker);+ await upsertOne(GuildThreadState.upsertAll(marker));await ban(s);expect(await remaining(post.id)).toBe(0);expect(await remaining(s.threadId)).toBe(0);@@ -161,30 +153,30 @@ describe('thread maintenance sweeps', () => {it('includes thread messages in user and guild harvests', async () => {const s = await setup();- const harvested = await harvestMessages(- getChannelRepository().messages,- createUserID(BigInt(s.member.userId)),- Date.now(),- null,- );+ const userArchive = await harvestUserArchive(harness, deps, createUserID(BigInt(s.member.userId)));expect(- harvested.channelMessagesMap- .get(s.threadId)- ?.map((message) => message.id)+ HarvestedMessagesSchema.parse(userArchive.readJson(`channels/${s.threadId}/messages.json`))+ .map((message) => message.id).sort(),).toEqual([...s.messageIds].sort());const admin = await setUserACLs(harness, await createTestAccount(harness), [AdminACLs.WILDCARD]);- await createBuilder(harness, admin.token).post(`/admin/guilds/${s.guildId}/archives`).body({}).execute();- const payload = jobs.find((job) => job.task === 'harvestGuildData')?.payload;- expect(payload).toBeDefined();- vi.spyOn(deps.storageService, 'uploadObjectFromFile').mockResolvedValue(undefined as never);- const writeFile = vi.spyOn(fs.promises, 'writeFile');- await harvestGuildData(payload!, createTestWorkerHelpers({attempt: {isLastAttempt: true}}));- const threadFile = writeFile.mock.calls.find(([file]) =>- String(file).endsWith(`channels/${s.threadId}/messages.json`),+ const requested = await createBuilder<{archive_id: string}>(harness, admin.token)+ .post(`/admin/guilds/${s.guildId}/archives`)+ .body({})+ .execute();+ const drained = await worker.drain();+ expect(drained.failed).toBe(0);+ const completed = await deps.adminArchiveRepository.findBySubjectAndArchiveId(+ 'guild',+ BigInt(s.guildId),+ BigInt(requested.archive_id),);- expect(threadFile).toBeDefined();- const written = JSON.parse(String(threadFile![1])) as Array<{id: string}>;- expect(written.map((message) => message.id).sort()).toEqual([...s.messageIds].sort());+ assert(completed?.storageKey);+ const guildArchive = await readStoredArchive(harness, completed.storageKey);+ expect(+ HarvestedMessagesSchema.parse(guildArchive.readJson(`channels/${s.threadId}/messages.json`))+ .map((message) => message.id)+ .sort(),+ ).toEqual([...s.messageIds].sort());});});