Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

test(api): read data export and sweep tests from stored archives

Hampus13429ebfb252
diff --git a/fluxer_api/src/api/test/UserArchiveTestUtils.ts b/fluxer_api/src/api/test/UserArchiveTestUtils.ts
new file mode 100644
index 000000000..c193095fb
--- /dev/null
+++ b/fluxer_api/src/api/test/UserArchiveTestUtils.ts
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import assert from 'node:assert/strict';
+import type {UserID} from '@app/api/BrandedTypes';
+import {Config} from '@app/api/Config';
+import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
+import {createTestWorkerHelpers} from '@app/api/test/SyncTaskWorkerService';
+import {UserHarvest} from '@app/api/user/UserHarvestModel';
+import harvestUserData from '@app/api/worker/tasks/HarvestUserData';
+import type {WorkerDependencies} from '@app/api/worker/WorkerDependencies';
+import {strFromU8, unzipSync} from 'fflate';
+import {z} from 'zod';
+
+export async function harvestUserArchive(harness: ApiTestHarness, deps: WorkerDependencies, userId: UserID) {
+ const harvestId = await deps.snowflakeService.generate();
+ await deps.userHarvestRepository.create(
+ new UserHarvest({
+ user_id: userId,
+ harvest_id: harvestId,
+ requested_at: new Date(),
+ started_at: null,
+ completed_at: null,
+ failed_at: null,
+ storage_key: null,
+ file_size: null,
+ progress_percent: 0,
+ progress_step: null,
+ error_message: null,
+ download_url_expires_at: null,
+ }),
+ );
+ await harvestUserData(
+ {userId: userId.toString(), harvestId: harvestId.toString()},
+ createTestWorkerHelpers({attempt: {isLastAttempt: true}}),
+ );
+ const completed = await deps.userHarvestRepository.findByUserAndHarvestId(userId, harvestId);
+ assert.equal(completed?.getStatus(), 'completed');
+ assert(completed?.storageKey);
+ return readStoredArchive(harness, completed.storageKey);
+}
+
+export async function readStoredArchive(harness: ApiTestHarness, storageKey: string) {
+ const zip = await harness.storageService.readObject(Config.s3.buckets.harvests, storageKey);
+ const entries = unzipSync(zip);
+ return {
+ entries,
+ readJson(name: string): unknown {
+ const bytes = entries[name];
+ assert(bytes, `Archive is missing ${name}`);
+ return JSON.parse(strFromU8(bytes));
+ },
+ };
+}
+
+export const HarvestedMessagesSchema = z.array(z.object({id: z.string(), content: z.string()}));
diff --git a/fluxer_api/src/api/user/tests/HarvestTestUtils.ts b/fluxer_api/src/api/user/tests/HarvestTestUtils.ts
index 8cb334ecf..280dc07ab 100644
--- a/fluxer_api/src/api/user/tests/HarvestTestUtils.ts
+++ b/fluxer_api/src/api/user/tests/HarvestTestUtils.ts
@@ -6,7 +6,6 @@ import {createBuilder} from '@app/api/test/TestRequestBuilder';
import type {UserHarvest} from '@app/api/user/UserHarvestModel';
import {UserHarvestRepository} from '@app/api/user/UserHarvestRepository';
import type {HarvestDownloadUrlResponse} from '@fluxer/schema/src/domains/user/UserHarvestSchemas';
-import {expect} from 'vitest';
interface HarvestRequestResponse {
harvest_id: string;
@@ -26,40 +25,12 @@ export async function fetchHarvestDownload(
.execute();
}
-export async function expectHarvestDownloadFailsWithError(
- harness: ApiTestHarness,
- token: string,
- harvestId: string,
- expectedCode: string,
-): Promise<void> {
- const {json} = await createBuilder<Record<string, unknown>>(harness, token)
- .get(`/users/@me/harvest/${harvestId}/download`)
- .expect(400)
- .executeWithResponse();
- const errorResponse = json as {
- code: string;
- message: string;
- };
- expect(errorResponse.code).toBe(expectedCode);
-}
-
export async function markHarvestCompleted(userId: string, harvestId: string, expiresAt: Date): Promise<void> {
const harvestRepository = new UserHarvestRepository();
const harvest = await claimHarvest(harvestRepository, userId, harvestId);
await harvestRepository.markAsCompleted(harvest, `test/${harvestId}.zip`, 1024n, expiresAt);
}
-export async function markHarvestFailed(userId: string, harvestId: string, errorMessage: string): Promise<void> {
- const harvestRepository = new UserHarvestRepository();
- const harvest = await claimHarvest(harvestRepository, userId, harvestId);
- await harvestRepository.markAsFailed(harvest, errorMessage);
-}
-
-export async function markHarvestStarted(userId: string, harvestId: string): Promise<void> {
- const harvestRepository = new UserHarvestRepository();
- await claimHarvest(harvestRepository, userId, harvestId);
-}
-
async function claimHarvest(
repository: UserHarvestRepository,
userId: string,
@@ -69,8 +40,3 @@ async function claimHarvest(
if (!harvest) throw new Error(`Harvest ${harvestId} for user ${userId} not found`);
return repository.markAsStarted(harvest);
}
-
-export async function findHarvest(userId: string, harvestId: string): Promise<UserHarvest | null> {
- const harvestRepository = new UserHarvestRepository();
- return harvestRepository.findByUserAndHarvestId(createUserID(BigInt(userId)), BigInt(harvestId));
-}
diff --git a/fluxer_api/src/api/worker/tests/ThreadMaintenanceSweeps.test.ts b/fluxer_api/src/api/worker/tests/ThreadMaintenanceSweeps.test.ts
index 7b177472c..ff21f97ef 100644
--- a/fluxer_api/src/api/worker/tests/ThreadMaintenanceSweeps.test.ts
+++ b/fluxer_api/src/api/worker/tests/ThreadMaintenanceSweeps.test.ts
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-import fs from 'node:fs';
+import assert from 'node:assert/strict';
import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {createChannelID, createGuildID, createUserID} from '@app/api/BrandedTypes';
import {acceptInvite, createChannel, createChannelInvite, createGuild} from '@app/api/channel/tests/ChannelTestUtils';
@@ -8,16 +8,15 @@ import {threadsRequest} from '@app/api/channel/tests/ThreadTestUtils';
import {deleteOneOrMany, upsertOne} from '@app/api/database/CassandraQueryExecution';
import {clearChannelThreadsTaintCacheForTesting, isTainted} from '@app/api/experiment/ChannelThreadsGate';
import {ensureSessionStarted} from '@app/api/message/tests/MessageTestUtils';
-import {getSnowflakeService} from '@app/api/middleware/ServiceRegistry';
+import {getSnowflakeService, setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';
import {getChannelRepository} from '@app/api/middleware/ServiceSingletons';
import {GuildThreadState} from '@app/api/Tables';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
-import {NoopWorkerService} from '@app/api/test/NoopWorkerService';
-import {createTestWorkerHelpers} from '@app/api/test/SyncTaskWorkerService';
+import {SyncTaskWorkerService} from '@app/api/test/SyncTaskWorkerService';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
+import {HarvestedMessagesSchema, harvestUserArchive, readStoredArchive} from '@app/api/test/UserArchiveTestUtils';
import deleteUserMessagesInGuildByTime from '@app/api/worker/tasks/DeleteUserMessagesInGuildByTime';
import harvestGuildData from '@app/api/worker/tasks/HarvestGuildData';
-import {harvestMessages} from '@app/api/worker/tasks/HarvestUserData';
import {clearWorkerDependencies, setWorkerDependencies} from '@app/api/worker/WorkerContext';
import {initializeWorkerDependencies, type WorkerDependencies} from '@app/api/worker/WorkerDependencies';
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
@@ -25,7 +24,7 @@ import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants';
import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas';
import type {ThreadChannelResponse} from '@fluxer/schema/src/domains/channel/ThreadRequestSchemas';
import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas';
-import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
+import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
interface Setup {
owner: TestAccount;
@@ -39,7 +38,7 @@ interface Setup {
describe('thread maintenance sweeps', () => {
let harness: ApiTestHarness;
let deps: WorkerDependencies;
- let jobs: Array<{task: string; payload: Record<string, unknown>}>;
+ let worker: SyncTaskWorkerService;
beforeAll(async () => {
harness = await createApiTestHarness();
@@ -48,25 +47,16 @@ describe('thread maintenance sweeps', () => {
beforeEach(async () => {
await harness.reset();
clearChannelThreadsTaintCacheForTesting();
+ worker = new SyncTaskWorkerService({deleteUserMessagesInGuildByTime, harvestGuildData}, {deferred: true});
+ setInjectedWorkerService(worker);
deps = await initializeWorkerDependencies(getSnowflakeService());
setWorkerDependencies(deps);
- jobs = [];
- const addJob = async (task: string, payload: Record<string, unknown>) => {
- jobs.push({task, payload});
- if (task === 'deleteUserMessagesInGuildByTime')
- await deleteUserMessagesInGuildByTime(payload, createTestWorkerHelpers({attempt: {isLastAttempt: true}}));
- return 0n;
- };
- vi.spyOn(NoopWorkerService.prototype, 'addJob').mockImplementation(addJob as () => Promise<bigint>);
- });
-
- afterEach(() => {
- vi.restoreAllMocks();
});
afterAll(async () => {
clearWorkerDependencies();
clearChannelThreadsTaintCacheForTesting();
+ setInjectedWorkerService(undefined);
await harness.shutdown();
});
@@ -109,6 +99,7 @@ describe('thread maintenance sweeps', () => {
.body({delete_message_seconds: 3600})
.expect(204)
.execute();
+ expect((await worker.drain()).failed).toBe(0);
}
it('removes thread messages inside the ban delete window and decrements the count', async () => {
@@ -143,7 +134,8 @@ describe('thread maintenance sweeps', () => {
await deleteOneOrMany(GuildThreadState.deleteByPk({guild_id: guildId}));
clearChannelThreadsTaintCacheForTesting();
expect(await isTainted(guildId)).toBe(false);
- await upsertOne(GuildThreadState.upsertAll(marker!));
+ assert(marker);
+ await upsertOne(GuildThreadState.upsertAll(marker));
await ban(s);
expect(await remaining(post.id)).toBe(0);
expect(await remaining(s.threadId)).toBe(0);
@@ -161,30 +153,30 @@ describe('thread maintenance sweeps', () => {
it('includes thread messages in user and guild harvests', async () => {
const s = await setup();
- const harvested = await harvestMessages(
- getChannelRepository().messages,
- createUserID(BigInt(s.member.userId)),
- Date.now(),
- null,
- );
+ const userArchive = await harvestUserArchive(harness, deps, createUserID(BigInt(s.member.userId)));
expect(
- harvested.channelMessagesMap
- .get(s.threadId)
- ?.map((message) => message.id)
+ HarvestedMessagesSchema.parse(userArchive.readJson(`channels/${s.threadId}/messages.json`))
+ .map((message) => message.id)
.sort(),
).toEqual([...s.messageIds].sort());
const admin = await setUserACLs(harness, await createTestAccount(harness), [AdminACLs.WILDCARD]);
- await createBuilder(harness, admin.token).post(`/admin/guilds/${s.guildId}/archives`).body({}).execute();
- const payload = jobs.find((job) => job.task === 'harvestGuildData')?.payload;
- expect(payload).toBeDefined();
- vi.spyOn(deps.storageService, 'uploadObjectFromFile').mockResolvedValue(undefined as never);
- const writeFile = vi.spyOn(fs.promises, 'writeFile');
- await harvestGuildData(payload!, createTestWorkerHelpers({attempt: {isLastAttempt: true}}));
- const threadFile = writeFile.mock.calls.find(([file]) =>
- String(file).endsWith(`channels/${s.threadId}/messages.json`),
+ const requested = await createBuilder<{archive_id: string}>(harness, admin.token)
+ .post(`/admin/guilds/${s.guildId}/archives`)
+ .body({})
+ .execute();
+ const drained = await worker.drain();
+ expect(drained.failed).toBe(0);
+ const completed = await deps.adminArchiveRepository.findBySubjectAndArchiveId(
+ 'guild',
+ BigInt(s.guildId),
+ BigInt(requested.archive_id),
);
- expect(threadFile).toBeDefined();
- const written = JSON.parse(String(threadFile![1])) as Array<{id: string}>;
- expect(written.map((message) => message.id).sort()).toEqual([...s.messageIds].sort());
+ assert(completed?.storageKey);
+ const guildArchive = await readStoredArchive(harness, completed.storageKey);
+ expect(
+ HarvestedMessagesSchema.parse(guildArchive.readJson(`channels/${s.threadId}/messages.json`))
+ .map((message) => message.id)
+ .sort(),
+ ).toEqual([...s.messageIds].sort());
});
});