A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
chore(expressions): remove the pack residue cleanup tool (#2288)
Hampusbdcf4b25c0f3
diff --git a/fluxer_api/package.json b/fluxer_api/package.jsonindex 86821f455..ade0d74e1 100644--- a/fluxer_api/package.json+++ b/fluxer_api/package.json@@ -8,8 +8,7 @@"typecheck": "tsgo --noEmit","dev": "tsx watch --clear-screen=false src/AppEntrypoint.ts","start": "tsx src/AppEntrypoint.ts",- "start:worker": "tsx src/WorkerEntrypoint.ts",- "cleanup:expression-pack-residue": "tsx src/ExpressionPackResidueEntrypoint.ts"+ "start:worker": "tsx src/WorkerEntrypoint.ts"},"dependencies": {"@atproto/api": "catalog:",diff --git a/fluxer_api/scripts/build.mjs b/fluxer_api/scripts/build.mjsindex 5247192e8..ad5af6bd8 100644--- a/fluxer_api/scripts/build.mjs+++ b/fluxer_api/scripts/build.mjs@@ -64,11 +64,7 @@ const workspacePlugin = {rmSync(OUT_DIR, {recursive: true, force: true});await build({- entryPoints: [- join(API_ROOT, 'src/AppEntrypoint.ts'),- join(API_ROOT, 'src/WorkerEntrypoint.ts'),- join(API_ROOT, 'src/ExpressionPackResidueEntrypoint.ts'),- ],+ entryPoints: [join(API_ROOT, 'src/AppEntrypoint.ts'), join(API_ROOT, 'src/WorkerEntrypoint.ts')],outdir: OUT_DIR,bundle: true,platform: 'node',diff --git a/fluxer_api/src/ExpressionPackResidueEntrypoint.ts b/fluxer_api/src/ExpressionPackResidueEntrypoint.tsdeleted file mode 100644index 4cbf2d751..000000000--- a/fluxer_api/src/ExpressionPackResidueEntrypoint.ts+++ /dev/null@@ -1,110 +0,0 @@-// SPDX-License-Identifier: AGPL-3.0-or-later--import '@app/Instrument';-import {initializeConfig} from '@app/api/Config';-import {setDatabaseQueryExecutor} from '@app/api/database/CassandraQueryExecution';-import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';-import {GuildRepository} from '@app/api/guild/repositories/GuildRepository';-import {ExpressionAssetPurger} from '@app/api/guild/services/content/ExpressionAssetPurger';-import {AssetDeletionQueue} from '@app/api/infrastructure/AssetDeletionQueue';-import {initializeLogger} from '@app/api/Logger';-import {- ExpressionPackResidueCleanup,- formatExpressionPackResidueReport,-} from '@app/api/maintenance/ExpressionPackResidueCleanup';-import {getKVClient} from '@app/api/middleware/ServiceRegistry';-import {Config} from '@app/Config';-import {Logger} from '@app/Logger';-import {BACKGROUND_READ_TIMEOUT_MS, initCassandra, shutdownCassandra} from '@pkgs/cassandra/src/Client';-import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';--const USAGE = `Usage: fluxer-api expression-pack-residue [--apply]--Deletes the guild_emojis and guild_stickers rows that the removed expression-packs feature wrote with guild_id set to a pack id, and queues their CDN and S3-assets for purge through the asset deletion queue.--Rows are only ever read from the guild_emojis and guild_stickers partitions-named by a pack id in expression_packs, so a real guild's expressions are never-read and never deleted. A pack id that also names a live guild is reported and-skipped. Running it again after a successful run is a no-op.--Options:- --apply Delete the rows and queue their assets. Without it the tool reports- what it would delete and changes nothing.- --help Show this message.`;--interface Options {- apply: boolean;- help: boolean;-}--function parseArgs(): Options {- const options: Options = {apply: false, help: false};- for (const arg of process.argv.slice(2)) {- if (arg === '--apply') {- options.apply = true;- } else if (arg === '--help' || arg === '-h') {- options.help = true;- } else {- throw new Error(`Unknown argument "${arg}". Run with --help for usage.`);- }- }- return options;-}--async function initializeDatabase(): Promise<() => Promise<void>> {- if (Config.database.backend === 'postgres') {- await initPostgres(Config.postgres);- const postgres = getDefaultPostgresClient();- await ensurePostgresKvSchema(postgres);- setDatabaseQueryExecutor(new PostgresKvQueryExecutor(postgres));- return async () => {- setDatabaseQueryExecutor(null);- await shutdownPostgres();- };- }- await initCassandra({- hosts: Config.cassandra.hosts.split(',').filter(Boolean),- port: Config.cassandra.port,- keyspace: Config.cassandra.keyspace,- localDc: Config.cassandra.localDc,- username: Config.cassandra.username || undefined,- password: Config.cassandra.password || undefined,- readTimeoutMs: BACKGROUND_READ_TIMEOUT_MS,- });- return async () => {- await shutdownCassandra();- };-}--async function main(): Promise<void> {- const options = parseArgs();- if (options.help) {- console.log(USAGE);- return;- }- initializeConfig(Config);- initializeLogger(Logger);- const shutdownDatabase = await initializeDatabase();- const kvClient = getKVClient();- try {- const cleanup = new ExpressionPackResidueCleanup(- new GuildRepository(),- new ExpressionAssetPurger(new AssetDeletionQueue(kvClient)),- );- const report = await cleanup.run({apply: options.apply});- console.log(formatExpressionPackResidueReport(report));- } finally {- await shutdownDatabase();- }-}--main()- .then(() => {- process.exit(0);- })- .catch((error) => {- Logger.fatal({error}, 'Expression pack residue cleanup failed');- process.exit(1);- });diff --git a/fluxer_api/src/api/maintenance/ExpressionPackResidueCleanup.test.ts b/fluxer_api/src/api/maintenance/ExpressionPackResidueCleanup.test.tsdeleted file mode 100644index 7ae933c0b..000000000--- a/fluxer_api/src/api/maintenance/ExpressionPackResidueCleanup.test.ts+++ /dev/null@@ -1,451 +0,0 @@-// SPDX-License-Identifier: AGPL-3.0-or-later--import {spawnSync} from 'node:child_process';-import {createServer} from 'node:net';-import {- getDefaultPostgresClient,- type IPostgresClient,- initPostgres,- shutdownPostgres,-} from '@pkgs/postgres/src/Client';-import {afterAll, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';-import type {EmojiID, GuildID, StickerID, UserID} from '../BrandedTypes';-import {createEmojiID, createGuildID, createStickerID, createUserID} from '../BrandedTypes';-import {setCassandraQueryExecutorForTesting, upsertOne} from '../database/CassandraQueryExecution';-import {defineTable} from '../database/CassandraTableDsl';-import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '../database/PostgresKvQueryExecutor';-import {GuildRepository} from '../guild/repositories/GuildRepository';-import {ExpressionAssetPurger} from '../guild/services/content/ExpressionAssetPurger';-import {AssetDeletionQueue} from '../infrastructure/AssetDeletionQueue';-import {getKVClient} from '../middleware/ServiceRegistry';-import {GuildEmoji} from '../models/GuildEmoji';-import {GuildSticker} from '../models/GuildSticker';-import {startDockerContainer} from '../test/DockerTestContainer';-import {InMemoryCassandraQueryExecutor} from '../test/InMemoryCassandraQueryExecutor';-import {- ExpressionPackResidueCleanup,- type ExpressionPackResidueRepository,- formatExpressionPackResidueReport,-} from './ExpressionPackResidueCleanup';--interface ExpressionPackSeedRow {- pack_id: GuildID;- pack_type: string;- creator_id: UserID;- name: string;- description: string | null;- created_at: Date;- updated_at: Date;- version: number;-}--const ExpressionPacksSeed = defineTable<ExpressionPackSeedRow, 'pack_id'>({- name: 'expression_packs',- columns: ['pack_id', 'pack_type', 'creator_id', 'name', 'description', 'created_at', 'updated_at', 'version'],- primaryKey: ['pack_id'],-});--const KV_TABLE = 'kv_pack_residue';-const CONTAINER = `fluxer-packresidue-${process.pid.toString(36)}-${Date.now().toString(36)}`;-const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;--const CREATOR_ID = createUserID(700000000000000001n);-const PACK_ID = createGuildID(900000000000000001n);-const SECOND_PACK_ID = createGuildID(900000000000000002n);-const GUILD_ID = createGuildID(100000000000000001n);-const PACK_ID_THAT_IS_A_GUILD = createGuildID(100000000000000002n);-const PACK_EMOJI_ID = createEmojiID(910000000000000001n);-const SECOND_PACK_EMOJI_ID = createEmojiID(910000000000000002n);-const PACK_STICKER_ID = createStickerID(920000000000000001n);-const GUILD_EMOJI_ID = createEmojiID(110000000000000001n);-const GUILD_STICKER_ID = createStickerID(120000000000000001n);--async function sleep(ms: number): Promise<void> {- await new Promise((resolve) => setTimeout(resolve, ms));-}--async function freePort(): Promise<number> {- return new Promise((resolve, reject) => {- const server = createServer();- server.on('error', reject);- server.listen(0, '127.0.0.1', () => {- const address = server.address();- if (typeof address === 'string' || address === null) {- reject(new Error('no port'));- return;- }- const port = address.port;- server.close(() => resolve(port));- });- });-}--function assetQueue(): AssetDeletionQueue {- return new AssetDeletionQueue(getKVClient());-}--function createCleanup(repository?: ExpressionPackResidueRepository): ExpressionPackResidueCleanup {- return new ExpressionPackResidueCleanup(repository ?? new GuildRepository(), new ExpressionAssetPurger(assetQueue()));-}--async function seedPack(packId: GuildID): Promise<void> {- await upsertOne(- ExpressionPacksSeed.insert({- pack_id: packId,- pack_type: 'emoji',- creator_id: CREATOR_ID,- name: `pack ${packId}`,- description: null,- created_at: new Date('2026-01-01T00:00:00.000Z'),- updated_at: new Date('2026-01-01T00:00:00.000Z'),- version: 1,- }),- );-}--async function seedGuild(guildId: GuildID): Promise<void> {- await new GuildRepository().upsertPartial(guildId, {name: `guild ${guildId}`, owner_id: CREATOR_ID});-}--async function seedEmoji(guildId: GuildID, emojiId: EmojiID, name: string): Promise<void> {- await new GuildRepository().upsertEmoji({- guild_id: guildId,- emoji_id: emojiId,- name,- creator_id: CREATOR_ID,- animated: false,- nsfw: null,- version: 1,- });-}--async function seedSticker(guildId: GuildID, stickerId: StickerID, name: string): Promise<void> {- await new GuildRepository().upsertSticker({- guild_id: guildId,- sticker_id: stickerId,- name,- description: null,- animated: false,- nsfw: null,- tags: null,- creator_id: CREATOR_ID,- version: 1,- });-}--async function emojiNames(guildId: GuildID): Promise<Array<string>> {- const emojis = await new GuildRepository().listEmojis(guildId);- return emojis.map((emoji) => emoji.name).sort();-}--async function stickerNames(guildId: GuildID): Promise<Array<string>> {- const stickers = await new GuildRepository().listStickers(guildId);- return stickers.map((sticker) => sticker.name).sort();-}--async function queuedS3Keys(): Promise<Array<string>> {- const queue = assetQueue();- const items = await queue.getBatch(100);- return items.map((item) => item.s3Key).filter((key) => key.length > 0);-}--function defineResidueSuite(resetData: () => Promise<void>): void {- beforeEach(async () => {- await resetData();- await assetQueue().clear();- });-- it('deletes every emoji and sticker owned by a pack id and queues their assets', async () => {- await seedPack(PACK_ID);- await seedEmoji(PACK_ID, PACK_EMOJI_ID, 'packblob');- await seedEmoji(PACK_ID, SECOND_PACK_EMOJI_ID, 'packwave');- await seedSticker(PACK_ID, PACK_STICKER_ID, 'packsticker');-- const report = await createCleanup().run({apply: true});-- expect(report.emojiCount).toBe(2);- expect(report.stickerCount).toBe(1);- expect(await emojiNames(PACK_ID)).toEqual([]);- expect(await stickerNames(PACK_ID)).toEqual([]);- expect(await new GuildRepository().getEmojiById(PACK_EMOJI_ID)).toBeNull();- expect(await new GuildRepository().getStickerById(PACK_STICKER_ID)).toBeNull();- expect(await queuedS3Keys()).toEqual([- `emojis/${PACK_EMOJI_ID}`,- `emojis/${SECOND_PACK_EMOJI_ID}`,- `stickers/${PACK_STICKER_ID}`,- ]);- });-- it('leaves a real guild emoji and sticker completely untouched', async () => {- await seedPack(PACK_ID);- await seedGuild(GUILD_ID);- await seedEmoji(GUILD_ID, GUILD_EMOJI_ID, 'guildblob');- await seedSticker(GUILD_ID, GUILD_STICKER_ID, 'guildsticker');-- const report = await createCleanup().run({apply: true});-- expect(report.residues).toEqual([]);- expect(report.emojiCount).toBe(0);- expect(report.stickerCount).toBe(0);- expect(await emojiNames(GUILD_ID)).toEqual(['guildblob']);- expect(await stickerNames(GUILD_ID)).toEqual(['guildsticker']);- expect(await new GuildRepository().getEmojiById(GUILD_EMOJI_ID)).not.toBeNull();- expect(await new GuildRepository().getStickerById(GUILD_STICKER_ID)).not.toBeNull();- expect(await queuedS3Keys()).toEqual([]);- });-- it('deletes only the pack rows out of a mixed dataset', async () => {- await seedPack(PACK_ID);- await seedPack(SECOND_PACK_ID);- await seedGuild(GUILD_ID);- await seedEmoji(PACK_ID, PACK_EMOJI_ID, 'packblob');- await seedSticker(PACK_ID, PACK_STICKER_ID, 'packsticker');- await seedEmoji(GUILD_ID, GUILD_EMOJI_ID, 'guildblob');- await seedSticker(GUILD_ID, GUILD_STICKER_ID, 'guildsticker');-- const report = await createCleanup().run({apply: true});-- expect(report.packIds).toEqual([PACK_ID, SECOND_PACK_ID]);- expect(report.residues.map((residue) => residue.packId)).toEqual([PACK_ID]);- expect(await emojiNames(PACK_ID)).toEqual([]);- expect(await stickerNames(PACK_ID)).toEqual([]);- expect(await emojiNames(GUILD_ID)).toEqual(['guildblob']);- expect(await stickerNames(GUILD_ID)).toEqual(['guildsticker']);- expect(await queuedS3Keys()).toEqual([`emojis/${PACK_EMOJI_ID}`, `stickers/${PACK_STICKER_ID}`]);- });-- it('is a clean no-op on an empty dataset', async () => {- const report = await createCleanup().run({apply: true});-- expect(report).toEqual({- applied: true,- packIds: [],- liveGuildPackIds: [],- residues: [],- emojiCount: 0,- stickerCount: 0,- });- expect(await queuedS3Keys()).toEqual([]);- });-- it('reports the rows it would delete without deleting anything by default', async () => {- await seedPack(PACK_ID);- await seedEmoji(PACK_ID, PACK_EMOJI_ID, 'packblob');- await seedSticker(PACK_ID, PACK_STICKER_ID, 'packsticker');-- const report = await createCleanup().run({apply: false});-- expect(formatExpressionPackResidueReport(report)).toBe(- [- 'expression pack residue cleanup: DRY RUN, nothing deleted and nothing queued',- 'pack ids read from expression_packs: 1',- `pack ${PACK_ID}: 1 emojis, 1 stickers`,- ` guild_emojis guild_id=${PACK_ID} emoji_id=${PACK_EMOJI_ID} name="packblob"`,- ` guild_stickers guild_id=${PACK_ID} sticker_id=${PACK_STICKER_ID} name="packsticker"`,- 'totals: 1 emojis, 1 stickers, 2 assets',- ].join('\n'),- );- expect(await emojiNames(PACK_ID)).toEqual(['packblob']);- expect(await stickerNames(PACK_ID)).toEqual(['packsticker']);- expect(await queuedS3Keys()).toEqual([]);- });-- it('is a no-op when run again after a real run', async () => {- await seedPack(PACK_ID);- await seedEmoji(PACK_ID, PACK_EMOJI_ID, 'packblob');- await seedSticker(PACK_ID, PACK_STICKER_ID, 'packsticker');-- await createCleanup().run({apply: true});- await assetQueue().clear();- const second = await createCleanup().run({apply: true});-- expect(second.packIds).toEqual([PACK_ID]);- expect(second.residues).toEqual([]);- expect(second.emojiCount).toBe(0);- expect(second.stickerCount).toBe(0);- expect(await queuedS3Keys()).toEqual([]);- });-- it('skips a pack id that also names a live guild and leaves its expressions alone', async () => {- await seedPack(PACK_ID_THAT_IS_A_GUILD);- await seedGuild(PACK_ID_THAT_IS_A_GUILD);- await seedEmoji(PACK_ID_THAT_IS_A_GUILD, GUILD_EMOJI_ID, 'guildblob');- await seedSticker(PACK_ID_THAT_IS_A_GUILD, GUILD_STICKER_ID, 'guildsticker');-- const report = await createCleanup().run({apply: true});-- expect(report.liveGuildPackIds).toEqual([PACK_ID_THAT_IS_A_GUILD]);- expect(report.residues).toEqual([]);- expect(await emojiNames(PACK_ID_THAT_IS_A_GUILD)).toEqual(['guildblob']);- expect(await stickerNames(PACK_ID_THAT_IS_A_GUILD)).toEqual(['guildsticker']);- expect(await queuedS3Keys()).toEqual([]);- expect(formatExpressionPackResidueReport(report)).toContain(- `skipped pack ${PACK_ID_THAT_IS_A_GUILD}: that id is a live guild, its expressions were never read`,- );- });-- it('refuses to delete when a pack partition hands back a row owned by another guild', async () => {- await seedPack(PACK_ID);- const deleteEmoji = vi.fn(async () => {});- const deleteSticker = vi.fn(async () => {});- const repository: ExpressionPackResidueRepository = {- listGuilds: async () => [],- listEmojis: async () => [- new GuildEmoji({- guild_id: GUILD_ID,- emoji_id: GUILD_EMOJI_ID,- name: 'guildblob',- creator_id: CREATOR_ID,- animated: false,- nsfw: null,- version: 1,- }),- ],- listStickers: async () => [],- deleteEmoji,- deleteSticker,- };-- await expect(createCleanup(repository).run({apply: true})).rejects.toThrow(- `guild_emojis partition ${PACK_ID} returned a row owned by guild ${GUILD_ID}`,- );- expect(deleteEmoji).not.toHaveBeenCalled();- expect(deleteSticker).not.toHaveBeenCalled();- expect(await queuedS3Keys()).toEqual([]);- });-- it('refuses to delete when a pack partition hands back a sticker owned by another guild', async () => {- await seedPack(PACK_ID);- const deleteSticker = vi.fn(async () => {});- const repository: ExpressionPackResidueRepository = {- listGuilds: async () => [],- listEmojis: async () => [],- listStickers: async () => [- new GuildSticker({- guild_id: GUILD_ID,- sticker_id: GUILD_STICKER_ID,- name: 'guildsticker',- description: null,- animated: false,- nsfw: null,- tags: null,- creator_id: CREATOR_ID,- version: 1,- }),- ],- deleteEmoji: async () => {},- deleteSticker,- };-- await expect(createCleanup(repository).run({apply: true})).rejects.toThrow(- `guild_stickers partition ${PACK_ID} returned a row owned by guild ${GUILD_ID}`,- );- expect(deleteSticker).not.toHaveBeenCalled();- });-}--describe('ExpressionPackResidueCleanup against the cassandra query shapes', () => {- const executor = new InMemoryCassandraQueryExecutor();-- beforeAll(() => {- setCassandraQueryExecutorForTesting(executor);- });-- afterAll(() => {- setCassandraQueryExecutorForTesting(new InMemoryCassandraQueryExecutor());- });-- defineResidueSuite(async () => {- executor.reset();- });-});--describe.skipIf(!dockerAvailable)('ExpressionPackResidueCleanup against the postgres kv backend', () => {- let raw: IPostgresClient;-- beforeAll(async () => {- const port = await freePort();- startDockerContainer([- 'run',- '-d',- '--name',- CONTAINER,- '-e',- 'POSTGRES_USER=fluxer',- '-e',- 'POSTGRES_PASSWORD=fluxer',- '-e',- 'POSTGRES_DB=fluxer',- '-p',- `127.0.0.1:${port}:5432`,- 'postgres:16-alpine',- '-c',- 'fsync=off',- ]);- let ready = false;- for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {- await sleep(500);- const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {- stdio: 'ignore',- });- if (probe.status !== 0) continue;- try {- await initPostgres({- url: `postgres://fluxer:[email protected]:${port}/fluxer`,- maxConnections: 1,- kvTable: KV_TABLE,- });- await getDefaultPostgresClient().query('SELECT 1');- ready = true;- } catch {- await shutdownPostgres().catch(() => {});- }- }- if (!ready) throw new Error('postgres never came up');- raw = getDefaultPostgresClient();- await ensurePostgresKvSchema(raw);- setCassandraQueryExecutorForTesting(new PostgresKvQueryExecutor(raw));- }, 900_000);-- afterAll(async () => {- setCassandraQueryExecutorForTesting(new InMemoryCassandraQueryExecutor());- await shutdownPostgres().catch(() => {});- spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});- });-- defineResidueSuite(async () => {- await raw.query(`DELETE FROM ${KV_TABLE}`);- });-});--describe('ExpressionPackResidueCleanup on a deployment without the pack tables', () => {- afterAll(() => {- setCassandraQueryExecutorForTesting(new InMemoryCassandraQueryExecutor());- });-- it('treats a missing expression_packs table as nothing to clean', async () => {- setCassandraQueryExecutorForTesting({- executeQuery: async () => {- throw new Error('unconfigured table expression_packs');- },- executeBatch: async () => {},- });-- const report = await createCleanup().run({apply: true});-- expect(report.packIds).toEqual([]);- expect(report.residues).toEqual([]);- expect(await queuedS3Keys()).toEqual([]);- });-- it('rethrows any other failure while reading expression_packs', async () => {- setCassandraQueryExecutorForTesting({- executeQuery: async () => {- throw new Error('Operation timed out');- },- executeBatch: async () => {},- });-- await expect(createCleanup().run({apply: true})).rejects.toThrow('Operation timed out');- });-});diff --git a/fluxer_api/src/api/maintenance/ExpressionPackResidueCleanup.ts b/fluxer_api/src/api/maintenance/ExpressionPackResidueCleanup.tsdeleted file mode 100644index 6209472bc..000000000--- a/fluxer_api/src/api/maintenance/ExpressionPackResidueCleanup.ts+++ /dev/null@@ -1,206 +0,0 @@-// SPDX-License-Identifier: AGPL-3.0-or-later--import type {EmojiID, GuildID, StickerID} from '../BrandedTypes';-import {fetchPage, type PagedQueryResult} from '../database/CassandraQueryExecution';-import {defineTable} from '../database/CassandraTableDsl';-import type {IGuildRepositoryAggregate} from '../guild/repositories/IGuildRepositoryAggregate';-import type {ExpressionAssetPurger} from '../guild/services/content/ExpressionAssetPurger';--const PACK_ID_PAGE_SIZE = 500;-const LIVE_GUILD_CHUNK_SIZE = 100;-const EXPRESSION_PACKS_TABLE = 'expression_packs';-const MISSING_TABLE_PATTERN = /unconfigured|undefined table|does not exist/i;--interface ExpressionPackIdRow {- pack_id: GuildID;-}--const ExpressionPacks = defineTable<ExpressionPackIdRow, 'pack_id'>({- name: EXPRESSION_PACKS_TABLE,- columns: ['pack_id'],- primaryKey: ['pack_id'],-});--const SCAN_EXPRESSION_PACK_IDS_QUERY = ExpressionPacks.selectCql();--export type ExpressionPackResidueRepository = Pick<- IGuildRepositoryAggregate,- 'listGuilds' | 'listEmojis' | 'listStickers' | 'deleteEmoji' | 'deleteSticker'->;--interface ExpressionPackResidueEmoji {- id: EmojiID;- name: string;-}--interface ExpressionPackResidueSticker {- id: StickerID;- name: string;-}--interface ExpressionPackResidue {- packId: GuildID;- emojis: Array<ExpressionPackResidueEmoji>;- stickers: Array<ExpressionPackResidueSticker>;-}--interface ExpressionPackResidueReport {- applied: boolean;- packIds: Array<GuildID>;- liveGuildPackIds: Array<GuildID>;- residues: Array<ExpressionPackResidue>;- emojiCount: number;- stickerCount: number;-}--function compareIds(left: bigint, right: bigint): number {- if (left === right) return 0;- return left < right ? -1 : 1;-}--function isMissingPackTableError(error: unknown): boolean {- const message = error instanceof Error ? error.message : String(error);- return message.includes(EXPRESSION_PACKS_TABLE) && MISSING_TABLE_PATTERN.test(message);-}--export class ExpressionPackResidueCleanup {- constructor(- private readonly guildRepository: ExpressionPackResidueRepository,- private readonly assetPurger: ExpressionAssetPurger,- ) {}-- async run(options: {apply: boolean}): Promise<ExpressionPackResidueReport> {- const packIds = await this.scanPackIds();- const report: ExpressionPackResidueReport = {- applied: options.apply,- packIds,- liveGuildPackIds: [],- residues: [],- emojiCount: 0,- stickerCount: 0,- };- if (packIds.length === 0) {- return report;- }- report.liveGuildPackIds = await this.findLiveGuildIds(packIds);- const liveGuildIds = new Set(report.liveGuildPackIds);- for (const packId of packIds) {- if (liveGuildIds.has(packId)) {- continue;- }- const residue = await this.collectResidue(packId);- if (residue.emojis.length === 0 && residue.stickers.length === 0) {- continue;- }- report.residues.push(residue);- report.emojiCount += residue.emojis.length;- report.stickerCount += residue.stickers.length;- if (options.apply) {- await this.deleteResidue(residue);- }- }- return report;- }-- private async scanPackIds(): Promise<Array<GuildID>> {- const packIds = new Set<GuildID>();- let pageState: string | null = null;- try {- do {- const page: PagedQueryResult<ExpressionPackIdRow> = await fetchPage<ExpressionPackIdRow>(- SCAN_EXPRESSION_PACK_IDS_QUERY,- {},- {pageSize: PACK_ID_PAGE_SIZE, pageState},- );- for (const row of page.rows) {- packIds.add(row.pack_id);- }- pageState = page.pageState;- } while (pageState !== null);- } catch (error) {- if (!isMissingPackTableError(error)) {- throw error;- }- return [];- }- return [...packIds].sort(compareIds);- }-- private async findLiveGuildIds(packIds: ReadonlyArray<GuildID>): Promise<Array<GuildID>> {- const liveGuildIds: Array<GuildID> = [];- for (let index = 0; index < packIds.length; index += LIVE_GUILD_CHUNK_SIZE) {- const guilds = await this.guildRepository.listGuilds(packIds.slice(index, index + LIVE_GUILD_CHUNK_SIZE));- for (const guild of guilds) {- liveGuildIds.push(guild.id);- }- }- return liveGuildIds.sort(compareIds);- }-- private async collectResidue(packId: GuildID): Promise<ExpressionPackResidue> {- const [emojis, stickers] = await Promise.all([- this.guildRepository.listEmojis(packId),- this.guildRepository.listStickers(packId),- ]);- for (const emoji of emojis) {- if (emoji.guildId !== packId) {- throw new Error(`guild_emojis partition ${packId} returned a row owned by guild ${emoji.guildId}`);- }- }- for (const sticker of stickers) {- if (sticker.guildId !== packId) {- throw new Error(`guild_stickers partition ${packId} returned a row owned by guild ${sticker.guildId}`);- }- }- return {- packId,- emojis: emojis- .map((emoji) => ({id: emoji.id, name: emoji.name}))- .sort((left, right) => compareIds(left.id, right.id)),- stickers: stickers- .map((sticker) => ({id: sticker.id, name: sticker.name}))- .sort((left, right) => compareIds(left.id, right.id)),- };- }-- private async deleteResidue(residue: ExpressionPackResidue): Promise<void> {- for (const emoji of residue.emojis) {- await this.guildRepository.deleteEmoji(residue.packId, emoji.id);- await this.assetPurger.purgeEmoji(emoji.id.toString());- }- for (const sticker of residue.stickers) {- await this.guildRepository.deleteSticker(residue.packId, sticker.id);- await this.assetPurger.purgeSticker(sticker.id.toString());- }- }-}--export function formatExpressionPackResidueReport(report: ExpressionPackResidueReport): string {- const lines: Array<string> = [- report.applied- ? 'expression pack residue cleanup: APPLIED, rows deleted and assets queued for purge'- : 'expression pack residue cleanup: DRY RUN, nothing deleted and nothing queued',- `pack ids read from expression_packs: ${report.packIds.length}`,- ];- for (const packId of report.liveGuildPackIds) {- lines.push(`skipped pack ${packId}: that id is a live guild, its expressions were never read`);- }- if (report.residues.length === 0) {- lines.push('no guild_emojis or guild_stickers rows are owned by a pack id');- }- for (const residue of report.residues) {- lines.push(`pack ${residue.packId}: ${residue.emojis.length} emojis, ${residue.stickers.length} stickers`);- for (const emoji of residue.emojis) {- lines.push(` guild_emojis guild_id=${residue.packId} emoji_id=${emoji.id} name=${JSON.stringify(emoji.name)}`);- }- for (const sticker of residue.stickers) {- lines.push(- ` guild_stickers guild_id=${residue.packId} sticker_id=${sticker.id} name=${JSON.stringify(sticker.name)}`,- );- }- }- lines.push(- `totals: ${report.emojiCount} emojis, ${report.stickerCount} stickers, ${report.emojiCount + report.stickerCount} assets`,- );- return lines.join('\n');-}