A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
refactor(api): purge cache by canonical media prefix (#2802)
Hampus7412ec3395b8
diff --git a/fluxer_api/pkgs/kv_client/src/IKVProvider.ts b/fluxer_api/pkgs/kv_client/src/IKVProvider.tsindex d54946c14..82e774165 100644--- a/fluxer_api/pkgs/kv_client/src/IKVProvider.ts+++ b/fluxer_api/pkgs/kv_client/src/IKVProvider.ts@@ -29,7 +29,7 @@ export interface IKVSubscription {}export interface KVPurgeBatchResult {- urls: Array<string>;+ entries: Array<string>;tokensConsumed: number;}diff --git a/fluxer_api/pkgs/kv_client/src/KVClient.ts b/fluxer_api/pkgs/kv_client/src/KVClient.tsindex 4e3b3ad96..8203e7b88 100644--- a/fluxer_api/pkgs/kv_client/src/KVClient.ts+++ b/fluxer_api/pkgs/kv_client/src/KVClient.ts@@ -217,7 +217,7 @@ local refillIntervalMs = tonumber(ARGV[5])local queueSize = redis.call('SCARD', queueKey)if queueSize == 0 then- return '{"urls":[],"tokens":0}'+ return '{"entries":[],"tokens":0}'endlocal tokens = maxTokens@@ -237,14 +237,14 @@ endlocal toPop = math.min(maxItems, math.floor(tokens), queueSize)if toPop <= 0 thenredis.call('SET', bucketKey, cjson.encode({tokens = tokens, lastRefill = lastRefill}), 'EX', 3600)- return '{"urls":[],"tokens":0}'+ return '{"entries":[],"tokens":0}'end-local urls = redis.call('SPOP', queueKey, toPop)-tokens = tokens - #urls+local entries = redis.call('SPOP', queueKey, toPop)+tokens = tokens - #entriesredis.call('SET', bucketKey, cjson.encode({tokens = tokens, lastRefill = lastRefill}), 'EX', 3600)-return cjson.encode({urls = urls, tokens = #urls})+return cjson.encode({entries = entries, tokens = #entries})`;const CLAIM_BULK_DELETION_SCRIPT = `local score = redis.call('ZSCORE', KEYS[1], ARGV[1])@@ -902,17 +902,17 @@ function parseRateLimitResult(value: unknown): KVRateLimitResult {function parsePurgeBatchResult(value: unknown, maxItems: number): KVPurgeBatchResult {const command = 'dequeuePurgeBatch';if (!isJsonObject(value)) throw createInvalidResponseError(command, 'a purge batch object');- const {urls, tokens} = value;+ const {entries, tokens} = value;if (- !Array.isArray(urls) ||- !urls.every((url): url is string => typeof url === 'string') ||+ !Array.isArray(entries) ||+ !entries.every((entry): entry is string => typeof entry === 'string') ||!isNonNegativeSafeInteger(tokens) ||- tokens !== urls.length ||- urls.length > maxItems+ tokens !== entries.length ||+ entries.length > maxItems) {throw createInvalidResponseError(command, 'a bounded string array and matching token count');}- return {urls, tokensConsumed: tokens};+ return {entries, tokensConsumed: tokens};}function isJsonObject(value: unknown): value is Record<string, unknown> {diff --git a/fluxer_api/pkgs/kv_client/src/__tests__/KVClient.test.ts b/fluxer_api/pkgs/kv_client/src/__tests__/KVClient.test.tsindex 8ddbb46c7..9f3b81116 100644--- a/fluxer_api/pkgs/kv_client/src/__tests__/KVClient.test.ts+++ b/fluxer_api/pkgs/kv_client/src/__tests__/KVClient.test.ts@@ -153,7 +153,7 @@ describe('KVClient script execution', () => {},{name: 'dequeuePurgeBatch',- reply: JSON.stringify({urls: ['https://fluxer.test/a.png'], tokens: 1}),+ reply: JSON.stringify({entries: ['/attachments/1/2/a'], tokens: 1}),keyCount: 2,run: async (client) => client.dequeuePurgeBatch('queue:key', 'bucket:key', 10, 10, 1, 1000),},diff --git a/fluxer_api/src/api/infrastructure/CachePurgeAdapter.ts b/fluxer_api/src/api/infrastructure/CachePurgeAdapter.tsindex 71b96dd40..7e881da59 100644--- a/fluxer_api/src/api/infrastructure/CachePurgeAdapter.ts+++ b/fluxer_api/src/api/infrastructure/CachePurgeAdapter.ts@@ -5,18 +5,13 @@ import {createHttpCachePurgeAdapter} from '@app/api/infrastructure/HttpCachePurgimport {createNoneCachePurgeAdapter} from '@app/api/infrastructure/NoneCachePurgeAdapter';import type {CachePurgeAdapterName} from '@fluxer/config/src/MasterConfig';-export interface CachePurgeBatch {- readonly exact: ReadonlyArray<string>;- readonly prefix: ReadonlyArray<string>;-}-export type CachePurgeOutcome =| {readonly kind: 'purged'}- | {readonly kind: 'invalid_entries'; readonly status: number}+ | {readonly kind: 'rejected'; readonly status: number}| {readonly kind: 'failed'; readonly status: number | null; readonly error: unknown};export interface CachePurgeAdapter {- purge(batch: CachePurgeBatch): Promise<CachePurgeOutcome>;+ purge(prefixes: ReadonlyArray<string>): Promise<CachePurgeOutcome>;}const CACHE_PURGE_ADAPTERS = {diff --git a/fluxer_api/src/api/infrastructure/CachePurgePaths.test.ts b/fluxer_api/src/api/infrastructure/CachePurgePaths.test.tsnew file mode 100644index 000000000..e84eaa8c6--- /dev/null+++ b/fluxer_api/src/api/infrastructure/CachePurgePaths.test.ts@@ -0,0 +1,70 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {Config} from '@app/api/Config';+import {canonicalizePurgeUrl} from '@app/api/infrastructure/CachePurgePaths';+import {afterEach, beforeEach, describe, expect, it} from 'vitest';++const MEDIA = 'https://media.test';++describe('canonicalizePurgeUrl', () => {+ let previousMedia: string;++ beforeEach(() => {+ previousMedia = Config.endpoints.media;+ Config.endpoints.media = MEDIA;+ });++ afterEach(() => {+ Config.endpoints.media = previousMedia;+ });++ it('qualifies every prefix with the media host and no scheme', () => {+ expect(canonicalizePurgeUrl(`${MEDIA}/attachments/1/2/photo.png`)).toEqual(['media.test/attachments/1/2/photo']);+ });++ it('drops the file extension so every served format is covered', () => {+ expect(canonicalizePurgeUrl(`${MEDIA}/emojis/1531309058777157632.webp`)).toEqual([+ 'media.test/emojis/1531309058777157632',+ ]);+ });++ it('pairs an asset hash with its animated spelling in both directions', () => {+ expect(canonicalizePurgeUrl(`${MEDIA}/avatars/1/a_b35cc3d3`)).toEqual([+ 'media.test/avatars/1/a_b35cc3d3',+ 'media.test/avatars/1/b35cc3d3',+ ]);+ expect(canonicalizePurgeUrl(`${MEDIA}/icons/1/808b11fa.webp`)).toEqual([+ 'media.test/icons/1/808b11fa',+ 'media.test/icons/1/a_808b11fa',+ ]);+ });++ it('leaves a non-hash stem alone', () => {+ expect(canonicalizePurgeUrl(`${MEDIA}/attachments/1/2/holiday.jpg`)).toEqual([+ 'media.test/attachments/1/2/holiday',+ ]);+ });++ it('drops the query and fragment and decodes the path', () => {+ expect(canonicalizePurgeUrl(`${MEDIA}/attachments/1/2/ação.png?size=128#x`)).toEqual([+ 'media.test/attachments/1/2/ação',+ ]);+ });++ it('keeps a base path when the media endpoint carries one', () => {+ Config.endpoints.media = `${MEDIA}/media`;+ expect(canonicalizePurgeUrl(`${MEDIA}/media/avatars/1/b35cc3d3`)).toEqual([+ 'media.test/media/avatars/1/b35cc3d3',+ 'media.test/media/avatars/1/a_b35cc3d3',+ ]);+ expect(canonicalizePurgeUrl(`${MEDIA}/avatars/1/b35cc3d3`)).toEqual([]);+ });++ it('refuses anything that is not a media CDN object', () => {+ expect(canonicalizePurgeUrl('https://elsewhere.test/avatars/1/b35cc3d3')).toEqual([]);+ expect(canonicalizePurgeUrl(`${MEDIA}/avatars`)).toEqual([]);+ expect(canonicalizePurgeUrl(`${MEDIA}/`)).toEqual([]);+ expect(canonicalizePurgeUrl(`${MEDIA}/emojis/.webp`)).toEqual([]);+ expect(canonicalizePurgeUrl('not-a-url')).toEqual([]);+ });+});diff --git a/fluxer_api/src/api/infrastructure/CachePurgePaths.ts b/fluxer_api/src/api/infrastructure/CachePurgePaths.tsnew file mode 100644index 000000000..891a4be71--- /dev/null+++ b/fluxer_api/src/api/infrastructure/CachePurgePaths.ts@@ -0,0 +1,53 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {Config} from '@app/api/Config';++const EXTENSION_PATTERN = /\.[A-Za-z0-9]{1,8}$/u;+const ASSET_HASH_PATTERN = /^[0-9a-f]{8}$/u;+const TRAILING_SLASHES_PATTERN = /\/+$/u;+const ANIMATED_PREFIX = 'a_';+const MIN_PATH_SEGMENTS = 2;++function mediaBase(): URL | null {+ return URL.parse(Config.endpoints.media);+}++function decodePathname(pathname: string): string {+ try {+ return decodeURIComponent(pathname);+ } catch {+ return pathname;+ }+}++function animationVariants(stem: string): Array<string> {+ if (stem.startsWith(ANIMATED_PREFIX)) {+ const bare = stem.slice(ANIMATED_PREFIX.length);+ return ASSET_HASH_PATTERN.test(bare) ? [stem, bare] : [stem];+ }+ return ASSET_HASH_PATTERN.test(stem) ? [stem, `${ANIMATED_PREFIX}${stem}`] : [stem];+}++export function canonicalizePurgeUrl(url: string): Array<string> {+ const base = mediaBase();+ const parsed = URL.parse(url);+ if (base === null || parsed === null || parsed.origin !== base.origin) {+ return [];+ }+ const basePath = base.pathname.replace(TRAILING_SLASHES_PATTERN, '');+ if (!parsed.pathname.startsWith(`${basePath}/`)) {+ return [];+ }+ const segments = decodePathname(parsed.pathname.slice(basePath.length))+ .split('/')+ .filter((segment) => segment !== '');+ if (segments.length < MIN_PATH_SEGMENTS) {+ return [];+ }+ const stem = segments[segments.length - 1]!.replace(EXTENSION_PATTERN, '');+ if (stem === '') {+ return [];+ }+ const directory = segments.slice(0, -1).join('/');+ return animationVariants(stem).map((variant) => `${base.host}${basePath}/${directory}/${variant}`);+}diff --git a/fluxer_api/src/api/infrastructure/CachePurgeQueue.ts b/fluxer_api/src/api/infrastructure/CachePurgeQueue.tsindex cdffe0b76..42a286d72 100644--- a/fluxer_api/src/api/infrastructure/CachePurgeQueue.ts+++ b/fluxer_api/src/api/infrastructure/CachePurgeQueue.ts@@ -1,6 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later-import type {CachePurgeBatch} from '@app/api/infrastructure/CachePurgeAdapter';+import {canonicalizePurgeUrl} from '@app/api/infrastructure/CachePurgePaths';import {Logger} from '@app/api/Logger';import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';@@ -8,112 +8,69 @@ export interface IPurgeQueue {addUrls(urls: Array<string>): Promise<void>;}-const EXACT_QUEUE_KEY = 'cache_purge:exact';-const PREFIX_QUEUE_KEY = 'cache_purge:prefix';-const EXACT_BUCKET_KEY = 'cache_purge:budget:exact';-const PREFIX_BUCKET_KEY = 'cache_purge:budget:prefix';+const QUEUE_KEY = 'cache_purge:queue';+const BUDGET_KEY = 'cache_purge:budget';const REJECTED_KEY = 'cache_purge:rejected';-const EXACT_MAX_TOKENS = 120;-const EXACT_REFILL_RATE = 5;-const EXACT_REFILL_INTERVAL_MS = 1000;-const PREFIX_MAX_TOKENS = 20;-const PREFIX_REFILL_RATE = 1;-const PREFIX_REFILL_INTERVAL_MS = 2000;--function isPrefix(url: string): boolean {- return url.endsWith('*') || url.endsWith('/');-}+const MAX_TOKENS = 120;+const REFILL_RATE = 5;+const REFILL_INTERVAL_MS = 1000;export class CachePurgeQueue implements IPurgeQueue {- private readonly kvClient: IKVProvider;-- constructor(kvClient: IKVProvider) {- this.kvClient = kvClient;- }+ constructor(private readonly kvClient: IKVProvider) {}async addUrls(urls: Array<string>): Promise<void> {- if (urls.length === 0) {- return;- }- const exactUrls: Array<string> = [];- const prefixUrls: Array<string> = [];+ const prefixes = new Set<string>();for (const url of urls) {- const trimmed = url.trim();- if (trimmed === '') {+ const canonical = canonicalizePurgeUrl(url);+ if (canonical.length === 0) {+ Logger.warn({url}, 'Skipped a cache purge URL that is not a media CDN object');continue;}- if (isPrefix(trimmed)) {- prefixUrls.push(trimmed);- } else {- exactUrls.push(trimmed);+ for (const prefix of canonical) {+ prefixes.add(prefix);}}+ if (prefixes.size === 0) {+ return;+ }try {- await this.addToSets({exact: exactUrls, prefix: prefixUrls});- Logger.debug({exact: exactUrls.length, prefix: prefixUrls.length}, 'Added URLs to cache purge queue');+ await this.kvClient.sadd(QUEUE_KEY, ...prefixes);+ Logger.debug({prefixes: prefixes.size}, 'Added prefixes to the cache purge queue');} catch (error) {- Logger.error(- {error, exact: exactUrls.length, prefix: prefixUrls.length},- 'Failed to add URLs to cache purge queue',- );+ Logger.error({error, prefixes: prefixes.size}, 'Failed to add prefixes to the cache purge queue');throw error;}}- async dequeueBatch(): Promise<CachePurgeBatch> {- const exact = await this.kvClient.dequeuePurgeBatch(- EXACT_QUEUE_KEY,- EXACT_BUCKET_KEY,- EXACT_MAX_TOKENS,- EXACT_MAX_TOKENS,- EXACT_REFILL_RATE,- EXACT_REFILL_INTERVAL_MS,+ async dequeueBatch(): Promise<Array<string>> {+ const batch = await this.kvClient.dequeuePurgeBatch(+ QUEUE_KEY,+ BUDGET_KEY,+ MAX_TOKENS,+ MAX_TOKENS,+ REFILL_RATE,+ REFILL_INTERVAL_MS,);- try {- const prefix = await this.kvClient.dequeuePurgeBatch(- PREFIX_QUEUE_KEY,- PREFIX_BUCKET_KEY,- PREFIX_MAX_TOKENS,- PREFIX_MAX_TOKENS,- PREFIX_REFILL_RATE,- PREFIX_REFILL_INTERVAL_MS,- );- return {exact: exact.urls, prefix: prefix.urls};- } catch (error) {- await this.requeue({exact: exact.urls, prefix: []});- throw error;- }+ return batch.entries;}- async requeue(batch: CachePurgeBatch): Promise<void> {+ async requeue(prefixes: ReadonlyArray<string>): Promise<void> {+ if (prefixes.length === 0) {+ return;+ }try {- await this.addToSets(batch);+ await this.kvClient.sadd(QUEUE_KEY, ...prefixes);} catch (error) {- Logger.error(- {error, exact: batch.exact.length, prefix: batch.prefix.length},- 'Failed to requeue cache purge entries',- );+ Logger.error({error, prefixes: prefixes.length}, 'Failed to requeue cache purge prefixes');throw error;}}- async reject(batch: CachePurgeBatch): Promise<void> {- await this.kvClient.sadd(REJECTED_KEY, ...batch.exact, ...batch.prefix);- Logger.error(- {exact: batch.exact.length, prefix: batch.prefix.length, key: REJECTED_KEY},- 'Set aside cache purge entries the endpoint rejected',- );- }-- private async addToSets(batch: CachePurgeBatch): Promise<void> {- const ops: Array<Promise<number>> = [];- if (batch.exact.length > 0) {- ops.push(this.kvClient.sadd(EXACT_QUEUE_KEY, ...batch.exact));- }- if (batch.prefix.length > 0) {- ops.push(this.kvClient.sadd(PREFIX_QUEUE_KEY, ...batch.prefix));+ async reject(prefixes: ReadonlyArray<string>): Promise<void> {+ if (prefixes.length === 0) {+ return;}- await Promise.all(ops);+ await this.kvClient.sadd(REJECTED_KEY, ...prefixes);}}diff --git a/fluxer_api/src/api/infrastructure/HttpCachePurgeAdapter.ts b/fluxer_api/src/api/infrastructure/HttpCachePurgeAdapter.tsindex 507821922..2af8c5c08 100644--- a/fluxer_api/src/api/infrastructure/HttpCachePurgeAdapter.ts+++ b/fluxer_api/src/api/infrastructure/HttpCachePurgeAdapter.ts@@ -1,26 +1,18 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport type {APICachePurgeConfig} from '@app/api/config/APIConfig';-import type {CachePurgeAdapter, CachePurgeBatch, CachePurgeOutcome} from '@app/api/infrastructure/CachePurgeAdapter';+import type {CachePurgeAdapter, CachePurgeOutcome} from '@app/api/infrastructure/CachePurgeAdapter';import * as FetchUtils from '@app/api/utils/FetchUtils';-function stripOneTrailingAsterisk(entry: string): string {- return entry.endsWith('*') ? entry.slice(0, -1) : entry;-}-export function createHttpCachePurgeAdapter(config: APICachePurgeConfig): CachePurgeAdapter {return {- async purge(batch: CachePurgeBatch): Promise<CachePurgeOutcome> {- const headers: Record<string, string> = {'Content-Type': 'application/json'};- if (config.http.token !== '') {- headers.Authorization = `Bearer ${config.http.token}`;- }+ async purge(prefixes: ReadonlyArray<string>): Promise<CachePurgeOutcome> {let response: Response;try {response = await fetch(config.http.endpoint, {method: 'POST',- headers,- body: JSON.stringify({exact: batch.exact, prefix: batch.prefix.map(stripOneTrailingAsterisk)}),+ headers: {'Content-Type': 'application/json', Authorization: `Bearer ${config.http.token}`},+ body: JSON.stringify({prefixes}),redirect: 'manual',signal: AbortSignal.timeout(config.http.timeoutMs),});@@ -32,7 +24,7 @@ export function createHttpCachePurgeAdapter(config: APICachePurgeConfig): CachePreturn {kind: 'purged'};}if (response.status === 400 || response.status === 422) {- return {kind: 'invalid_entries', status: response.status};+ return {kind: 'rejected', status: response.status};}return {kind: 'failed', status: response.status, error: null};},diff --git a/fluxer_api/src/api/test/mocks/MockKVProvider.ts b/fluxer_api/src/api/test/mocks/MockKVProvider.tsindex 64f5e22c9..18c2b6ce4 100644--- a/fluxer_api/src/api/test/mocks/MockKVProvider.ts+++ b/fluxer_api/src/api/test/mocks/MockKVProvider.ts@@ -704,7 +704,7 @@ export class MockKVProvider implements IKVProvider {refillRate: number,refillIntervalMs: number,): Promise<{- urls: Array<string>;+ entries: Array<string>;tokensConsumed: number;}> {this.dequeuePurgeBatchSpy(queueKey, bucketKey, maxItems, maxTokens, refillRate, refillIntervalMs);@@ -734,13 +734,13 @@ export class MockKVProvider implements IKVProvider {if (toPop <= 0) {this.stringStore.set(bucketKey, JSON.stringify({tokens, lastRefill}));this.expiries.set(bucketKey, now + 3600 * 1000);- return {urls: [], tokensConsumed: 0};+ return {entries: [], tokensConsumed: 0};}- const urls = await this.spop(queueKey, toPop);- tokens -= urls.length;+ const entries = await this.spop(queueKey, toPop);+ tokens -= entries.length;this.stringStore.set(bucketKey, JSON.stringify({tokens, lastRefill}));this.expiries.set(bucketKey, now + 3600 * 1000);- return {urls, tokensConsumed: urls.length};+ return {entries, tokensConsumed: entries.length};}async evalScript(diff --git a/fluxer_api/src/api/worker/tasks/ProcessCachePurgeQueue.ts b/fluxer_api/src/api/worker/tasks/ProcessCachePurgeQueue.tsindex 94880b951..670a5a45b 100644--- a/fluxer_api/src/api/worker/tasks/ProcessCachePurgeQueue.ts+++ b/fluxer_api/src/api/worker/tasks/ProcessCachePurgeQueue.ts@@ -1,104 +1,53 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport {Config} from '@app/api/Config';-import {- type CachePurgeBatch,- type CachePurgeOutcome,- createCachePurgeAdapter,-} from '@app/api/infrastructure/CachePurgeAdapter';+import {type CachePurgeOutcome, createCachePurgeAdapter} from '@app/api/infrastructure/CachePurgeAdapter';import {CachePurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';import {Logger} from '@app/api/Logger';import {getWorkerDependencies} from '@app/api/worker/WorkerContext';-import type {CachePurgeAdapterName} from '@fluxer/config/src/MasterConfig';import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';-const RUN_DEADLINE_MS = 10_000;-type CachePurgeFailure = Extract<CachePurgeOutcome, {kind: 'failed'}>;-function describeBatch(adapter: CachePurgeAdapterName, batch: CachePurgeBatch) {- return {adapter, exact: batch.exact.length, prefix: batch.prefix.length};-}--function logFailure(adapter: CachePurgeAdapterName, requeued: CachePurgeBatch, failure: CachePurgeFailure): void {+function logFailure(context: Record<string, unknown>, failure: CachePurgeFailure): void {const {status, error} = failure;- const context = {...describeBatch(adapter, requeued), status, error};if (status === null || status === 408 || status === 429 || status >= 500) {- Logger.warn(context, 'Cache purge request failed, requeued its entries');+ Logger.warn({...context, status, error}, 'Cache purge request failed, requeued its prefixes');return;}- Logger.error(context, 'Cache purge endpoint refused the request, requeued its entries');-}--function untriedFrom(singles: ReadonlyArray<CachePurgeBatch>, index: number): CachePurgeBatch {- const untried = singles.slice(index);- return {exact: untried.flatMap((single) => single.exact), prefix: untried.flatMap((single) => single.prefix)};+ Logger.error({...context, status, error}, 'Cache purge endpoint refused the request, requeued its prefixes');}const processCachePurgeQueue: WorkerTaskHandler = async (_payload, _helpers) => {- const adapterName = Config.cachePurge.adapter;- if (adapterName === 'none') {+ const adapter = Config.cachePurge.adapter;+ if (adapter === 'none') {Logger.warn('Skipped a cache purge run because the adapter is none');return;}- const adapter = createCachePurgeAdapter(Config.cachePurge);const queue = new CachePurgeQueue(getWorkerDependencies().kvClient);- const startedAt = Date.now();- const batch = await queue.dequeueBatch();- if (batch.exact.length === 0 && batch.prefix.length === 0) {+ const prefixes = await queue.dequeueBatch();+ if (prefixes.length === 0) {return;}+ const context = {adapter, prefixes: prefixes.length};let outcome: CachePurgeOutcome;try {- outcome = await adapter.purge(batch);+ outcome = await createCachePurgeAdapter(Config.cachePurge).purge(prefixes);} catch (error) {- await queue.requeue(batch);+ await queue.requeue(prefixes);throw error;}if (outcome.kind === 'purged') {- Logger.debug(describeBatch(adapterName, batch), 'Purged a batch of cache entries');+ Logger.debug(context, 'Purged a batch of cache prefixes');return;}- if (outcome.kind === 'failed') {- await queue.requeue(batch);- logFailure(adapterName, batch, outcome);+ if (outcome.kind === 'rejected') {+ await queue.reject(prefixes);+ Logger.error({...context, status: outcome.status}, 'Cache purge endpoint rejected the batch, set it aside');return;}- Logger.warn(- {...describeBatch(adapterName, batch), status: outcome.status},- 'Cache purge endpoint rejected the batch, retrying each entry on its own',- );- const singles: Array<CachePurgeBatch> = [- ...batch.exact.map((entry) => ({exact: [entry], prefix: []})),- ...batch.prefix.map((entry) => ({exact: [], prefix: [entry]})),- ];- for (const [index, single] of singles.entries()) {- if (Date.now() - startedAt >= RUN_DEADLINE_MS) {- const untried = untriedFrom(singles, index);- await queue.requeue(untried);- Logger.warn(- describeBatch(adapterName, untried),- 'Cache purge run reached its deadline, requeued untried entries',- );- return;- }- let singleOutcome: CachePurgeOutcome;- try {- singleOutcome = await adapter.purge(single);- if (singleOutcome.kind === 'invalid_entries') {- await queue.reject(single);- }- } catch (error) {- await queue.requeue(untriedFrom(singles, index));- throw error;- }- if (singleOutcome.kind === 'failed') {- const untried = untriedFrom(singles, index);- await queue.requeue(untried);- logFailure(adapterName, untried, singleOutcome);- return;- }- }+ await queue.requeue(prefixes);+ logFailure(context, outcome);};export default processCachePurgeQueue;diff --git a/fluxer_api/src/api/worker/tests/ProcessCachePurgeQueue.test.ts b/fluxer_api/src/api/worker/tests/ProcessCachePurgeQueue.test.tsindex b628de50c..e9acf2c64 100644--- a/fluxer_api/src/api/worker/tests/ProcessCachePurgeQueue.test.ts+++ b/fluxer_api/src/api/worker/tests/ProcessCachePurgeQueue.test.ts@@ -13,15 +13,14 @@ import {delay, HttpResponse, http} from 'msw';import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';const HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;-const ENDPOINT = 'https://cache-purge.test/purge';+const ENDPOINT = 'https://cache-purge.test/__cache/purge';const MEDIA = 'https://media.test';-const EXACT_KEY = 'cache_purge:exact';-const PREFIX_KEY = 'cache_purge:prefix';+const QUEUE_KEY = 'cache_purge:queue';+const BUDGET_KEY = 'cache_purge:budget';const REJECTED_KEY = 'cache_purge:rejected';interface PurgeBody {- exact: Array<string>;- prefix: Array<string>;+ prefixes: Array<string>;}interface RecordedRequest {@@ -52,10 +51,6 @@ function recordPurges(respond: (body: PurgeBody) => Response | Promise<Response>return requests;}-function entryCount(body: PurgeBody): number {- return body.exact.length + body.prefix.length;-}-async function members(kvClient: MockKVProvider, key: string): Promise<Array<string>> {return (await kvClient.smembers(key)).sort();}@@ -66,9 +61,12 @@ function sorted(values: Array<string>): Array<string> {describe('processCachePurgeQueue', () => {let previousCachePurge: APICachePurgeConfig;+ let previousMedia: string;beforeEach(() => {previousCachePurge = {adapter: Config.cachePurge.adapter, http: Config.cachePurge.http};+ previousMedia = Config.endpoints.media;+ Config.endpoints.media = MEDIA;Config.cachePurge.adapter = 'http';Config.cachePurge.http = {endpoint: ENDPOINT, token: 'test-token', timeoutMs: 50};});@@ -76,15 +74,18 @@ describe('processCachePurgeQueue', () => {afterEach(() => {Config.cachePurge.adapter = previousCachePurge.adapter;Config.cachePurge.http = previousCachePurge.http;+ Config.endpoints.media = previousMedia;clearWorkerDependencies();vi.useRealTimers();});- it('sends queued exact and prefix entries in one request and drains the queue', async () => {+ it('sends host qualified prefixes in one request and drains the queue', async () => {const {kvClient, queue} = createHarness();- const exact = [`${MEDIA}/avatars/1/a_abc`, `${MEDIA}/emojis/9.webp`, `${MEDIA}/attachments/1/2/ação.png`];- const prefix = [`${MEDIA}/some/dir/`];- await queue.addUrls([...exact, ...prefix]);+ await queue.addUrls([+ `${MEDIA}/avatars/1/a_b35cc3d3`,+ `${MEDIA}/emojis/9.webp`,+ `${MEDIA}/attachments/1/2/ação.png`,+ ]);const requests = recordPurges(() => new HttpResponse(null, {status: 204}));await processCachePurgeQueue({}, HELPERS);@@ -92,55 +93,69 @@ describe('processCachePurgeQueue', () => {expect(requests).toHaveLength(1);expect(requests[0]!.authorization).toBe('Bearer test-token');expect(requests[0]!.contentType).toBe('application/json');- expect(sorted(requests[0]!.body.exact)).toEqual(sorted(exact));- expect(requests[0]!.body.prefix).toEqual(prefix);- expect(await members(kvClient, EXACT_KEY)).toEqual([]);- expect(await members(kvClient, PREFIX_KEY)).toEqual([]);+ expect(sorted(requests[0]!.body.prefixes)).toEqual(+ sorted([+ 'media.test/avatars/1/a_b35cc3d3',+ 'media.test/avatars/1/b35cc3d3',+ 'media.test/emojis/9',+ 'media.test/attachments/1/2/ação',+ ]),+ );+ expect(await members(kvClient, QUEUE_KEY)).toEqual([]);});- it('strips a trailing asterisk from a prefix entry and keeps a trailing slash', async () => {- const {kvClient, queue} = createHarness();- await queue.addUrls([`${MEDIA}/stickers/1**`, `${MEDIA}/emojis/`]);+ it('collapses every extension of one asset into a single prefix', async () => {+ const {queue} = createHarness();+ await queue.addUrls([`${MEDIA}/stickers/7.webp`, `${MEDIA}/stickers/7.gif`, `${MEDIA}/stickers/7.png`]);const requests = recordPurges(() => new HttpResponse(null, {status: 204}));await processCachePurgeQueue({}, HELPERS);expect(requests).toHaveLength(1);- expect(requests[0]!.body.exact).toEqual([]);- expect(sorted(requests[0]!.body.prefix)).toEqual([`${MEDIA}/emojis/`, `${MEDIA}/stickers/1*`]);- expect(await members(kvClient, PREFIX_KEY)).toEqual([]);+ expect(requests[0]!.body.prefixes).toEqual(['media.test/stickers/7']);});- it('sends no authorization header when no token is configured', async () => {- Config.cachePurge.http.token = '';- const {queue} = createHarness();- await queue.addUrls([`${MEDIA}/avatars/1/abc`]);- const requests = recordPurges(() => new HttpResponse(null, {status: 204}));+ it.each([200, 202, 204])('treats %i as accepted and drops the batch', async (status) => {+ const {kvClient, queue} = createHarness();+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);+ const requests = recordPurges(() => new HttpResponse(null, {status}));await processCachePurgeQueue({}, HELPERS);expect(requests).toHaveLength(1);- expect(requests[0]!.authorization).toBeNull();+ expect(await members(kvClient, QUEUE_KEY)).toEqual([]);+ expect(await members(kvClient, REJECTED_KEY)).toEqual([]);+ });++ it('never queues a URL that is not a media CDN object', async () => {+ const {kvClient, queue} = createHarness();+ await queue.addUrls(['https://elsewhere.test/avatars/1/b35cc3d3', `${MEDIA}/avatars`, 'not-a-url']);+ const requests = recordPurges(() => new HttpResponse(null, {status: 204}));++ await processCachePurgeQueue({}, HELPERS);++ expect(requests).toEqual([]);+ expect(await members(kvClient, QUEUE_KEY)).toEqual([]);});it('requeues the whole batch after a server error', async () => {const {kvClient, queue} = createHarness();- const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];- const prefix = [`${MEDIA}/some/dir/`];- await queue.addUrls([...exact, ...prefix]);+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`, `${MEDIA}/attachments/1/3/b.png`]);const requests = recordPurges(() => new HttpResponse(null, {status: 503}));await processCachePurgeQueue({}, HELPERS);expect(requests).toHaveLength(1);- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));- expect(await members(kvClient, PREFIX_KEY)).toEqual(prefix);+ expect(await members(kvClient, QUEUE_KEY)).toEqual([+ 'media.test/attachments/1/2/a',+ 'media.test/attachments/1/3/b',+ ]);+ expect(await members(kvClient, REJECTED_KEY)).toEqual([]);});it('requeues the batch when the endpoint does not answer in time', async () => {const {kvClient, queue} = createHarness();- const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];- await queue.addUrls(exact);+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);const requests = recordPurges(async () => {await delay('infinite');return new HttpResponse(null, {status: 204});@@ -149,25 +164,23 @@ describe('processCachePurgeQueue', () => {await processCachePurgeQueue({}, HELPERS);expect(requests).toHaveLength(1);- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));+ expect(await members(kvClient, QUEUE_KEY)).toEqual(['media.test/attachments/1/2/a']);});it('requeues the batch when the endpoint is unreachable', async () => {const {kvClient, queue} = createHarness();- const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];- await queue.addUrls(exact);+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);const requests = recordPurges(() => HttpResponse.error());await processCachePurgeQueue({}, HELPERS);expect(requests).toHaveLength(1);- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));+ expect(await members(kvClient, QUEUE_KEY)).toEqual(['media.test/attachments/1/2/a']);});it('requeues the batch on a redirect without following it', async () => {const {kvClient, queue} = createHarness();- const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];- await queue.addUrls(exact);+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);const redirectTarget = 'https://cache-purge.test/elsewhere';const redirectedRequests: Array<string> = [];server.use(@@ -182,122 +195,51 @@ describe('processCachePurgeQueue', () => {expect(requests).toHaveLength(1);expect(redirectedRequests).toEqual([]);- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));+ expect(await members(kvClient, QUEUE_KEY)).toEqual(['media.test/attachments/1/2/a']);});- it('requeues the batch on an authorisation failure without splitting it', async () => {+ it('requeues rather than discards when the bearer token is refused', async () => {const {kvClient, queue} = createHarness();- const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`, `${MEDIA}/emojis/9.webp`];- await queue.addUrls(exact);+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);const requests = recordPurges(() => new HttpResponse(null, {status: 401}));await processCachePurgeQueue({}, HELPERS);expect(requests).toHaveLength(1);- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));+ expect(await members(kvClient, QUEUE_KEY)).toEqual(['media.test/attachments/1/2/a']);expect(await members(kvClient, REJECTED_KEY)).toEqual([]);});- it('sets aside only the entry the endpoint rejects on its own', async () => {+ it.each([400, 422])('sets the batch aside when the endpoint answers %i', async (status) => {const {kvClient, queue} = createHarness();- const poison = `${MEDIA}/attachments/1/2/poison.png`;- await queue.addUrls([`${MEDIA}/avatars/1/abc`, poison, `${MEDIA}/emojis/9.webp`]);- const requests = recordPurges((body) =>- body.exact.includes(poison) ? new HttpResponse(null, {status: 422}) : new HttpResponse(null, {status: 204}),- );+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`, `${MEDIA}/emojis/9.webp`]);+ const requests = recordPurges(() => new HttpResponse(null, {status}));await processCachePurgeQueue({}, HELPERS);- expect(requests.map((request) => entryCount(request.body))).toEqual([3, 1, 1, 1]);- expect(await members(kvClient, REJECTED_KEY)).toEqual([poison]);- expect(await members(kvClient, EXACT_KEY)).toEqual([]);- });-- it('splits the batch when the endpoint answers 400', async () => {- const {kvClient, queue} = createHarness();- const poison = `${MEDIA}/attachments/1/2/poison.png`;- await queue.addUrls([`${MEDIA}/avatars/1/abc`, poison, `${MEDIA}/emojis/9.webp`]);- const requests = recordPurges((body) =>- body.exact.includes(poison) ? new HttpResponse(null, {status: 400}) : new HttpResponse(null, {status: 204}),- );-- await processCachePurgeQueue({}, HELPERS);-- expect(requests.map((request) => entryCount(request.body))).toEqual([3, 1, 1, 1]);- expect(await members(kvClient, REJECTED_KEY)).toEqual([poison]);- expect(await members(kvClient, EXACT_KEY)).toEqual([]);- });-- it('keeps untried exact and prefix entries queued when a single-entry retry fails', async () => {- const {kvClient, queue} = createHarness();- const first = `${MEDIA}/avatars/1/abc`;- const second = `${MEDIA}/banners/1/def`;- const third = `${MEDIA}/emojis/9.webp`;- const directory = `${MEDIA}/some/dir/`;- await queue.addUrls([first, second, third, directory]);- let singleRequests = 0;- const requests = recordPurges((body) => {- if (entryCount(body) > 1) {- return new HttpResponse(null, {status: 422});- }- singleRequests++;- return new HttpResponse(null, {status: singleRequests === 1 ? 204 : 503});- });-- await processCachePurgeQueue({}, HELPERS);-- expect(requests.map((request) => request.body.exact)).toEqual([[first, second, third], [first], [second]]);- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted([second, third]));- expect(await members(kvClient, PREFIX_KEY)).toEqual([directory]);- expect(await members(kvClient, REJECTED_KEY)).toEqual([]);- });-- it('requeues untried entries when the fallback runs out of time', async () => {- vi.useFakeTimers({toFake: ['Date']});- vi.setSystemTime(new Date('2026-09-13T00:00:00.000Z'));- const {kvClient, queue} = createHarness();- const entries = [1, 2, 3, 4, 5].map((index) => `${MEDIA}/avatars/${index}/abc`);- await queue.addUrls(entries);- const requests = recordPurges((body) => {- if (entryCount(body) > 1) {- return new HttpResponse(null, {status: 422});- }- vi.setSystemTime(Date.now() + 4_000);- return new HttpResponse(null, {status: 204});- });-- await processCachePurgeQueue({}, HELPERS);-- expect(requests.map((request) => request.body.exact)).toEqual([entries, [entries[0]], [entries[1]], [entries[2]]]);- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted([entries[3]!, entries[4]!]));- expect(await members(kvClient, REJECTED_KEY)).toEqual([]);+ expect(requests).toHaveLength(1);+ expect(await members(kvClient, QUEUE_KEY)).toEqual([]);+ expect(await members(kvClient, REJECTED_KEY)).toEqual(['media.test/attachments/1/2/a', 'media.test/emojis/9']);});it('leaves the queue untouched when the adapter is none', async () => {Config.cachePurge.adapter = 'none';const {kvClient, queue} = createHarness();- const exact = [`${MEDIA}/avatars/1/abc`];- const prefix = [`${MEDIA}/some/dir/`];- await queue.addUrls([...exact, ...prefix]);+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);const requests = recordPurges(() => new HttpResponse(null, {status: 204}));await processCachePurgeQueue({}, HELPERS);expect(requests).toEqual([]);- expect(await members(kvClient, EXACT_KEY)).toEqual(exact);- expect(await members(kvClient, PREFIX_KEY)).toEqual(prefix);- expect(await kvClient.get('cache_purge:budget:exact')).toBeNull();- expect(await kvClient.get('cache_purge:budget:prefix')).toBeNull();+ expect(await members(kvClient, QUEUE_KEY)).toEqual(['media.test/attachments/1/2/a']);+ expect(await kvClient.get(BUDGET_KEY)).toBeNull();});it('sends no more than the token bucket allows and refills at the configured rate', async () => {vi.useFakeTimers({toFake: ['Date']});- vi.setSystemTime(new Date('2026-09-13T00:00:00.000Z'));+ vi.setSystemTime(new Date('2026-09-16T00:00:00.000Z'));const {kvClient, queue} = createHarness();- await queue.addUrls([- ...Array.from({length: 200}, (_, index) => `${MEDIA}/avatars/${index}/abc`),- ...Array.from({length: 30}, (_, index) => `${MEDIA}/dirs/${index}/`),- ]);+ await queue.addUrls(Array.from({length: 200}, (_, index) => `${MEDIA}/attachments/1/${index}/a.png`));const requests = recordPurges(() => new HttpResponse(null, {status: 204}));await processCachePurgeQueue({}, HELPERS);@@ -305,11 +247,7 @@ describe('processCachePurgeQueue', () => {vi.setSystemTime(Date.now() + 10_000);await processCachePurgeQueue({}, HELPERS);- expect(requests.map((request) => [request.body.exact.length, request.body.prefix.length])).toEqual([- [120, 20],- [50, 5],- ]);- expect(await kvClient.scard(EXACT_KEY)).toBe(30);- expect(await kvClient.scard(PREFIX_KEY)).toBe(5);+ expect(requests.map((request) => request.body.prefixes.length)).toEqual([120, 50]);+ expect(await kvClient.scard(QUEUE_KEY)).toBe(30);});});diff --git a/packages/config/src/ConfigLoader.ts b/packages/config/src/ConfigLoader.tsindex 7d173bd4d..b3e1ca9f5 100644--- a/packages/config/src/ConfigLoader.ts+++ b/packages/config/src/ConfigLoader.ts@@ -484,7 +484,8 @@ function validateCachePurgeConfig(config: MasterConfig): void {) {throw new Error('FLUXER_CACHE_PURGE_HTTP_ENDPOINT must be an absolute http or https URL without credentials');}- if (!/^[\x21-\x7e]*$/u.test(cachePurge.http.token)) {+ requireString(cachePurge.http.token, 'FLUXER_CACHE_PURGE_HTTP_TOKEN');+ if (!/^[\x21-\x7e]+$/u.test(cachePurge.http.token)) {throw new Error('FLUXER_CACHE_PURGE_HTTP_TOKEN must contain only visible ASCII characters');}assertIntegerInRange(cachePurge.http.timeout_ms, 'FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS', 1_000, 10_000);diff --git a/packages/config/src/__tests__/ConfigLoader.test.ts b/packages/config/src/__tests__/ConfigLoader.test.tsindex 3b1eb6242..fb2798968 100644--- a/packages/config/src/__tests__/ConfigLoader.test.ts+++ b/packages/config/src/__tests__/ConfigLoader.test.ts@@ -572,6 +572,14 @@ describe('ConfigLoader', () => {await expect(loadConfig()).rejects.toThrow('FLUXER_CACHE_PURGE_HTTP_ENDPOINT is required');});+ test('rejects the http cache purge adapter without a token', async () => {+ stubMinimalEnv({+ FLUXER_CACHE_PURGE_ADAPTER: 'http',+ FLUXER_CACHE_PURGE_HTTP_ENDPOINT: 'https://purge.internal/purge',+ });+ await expect(loadConfig()).rejects.toThrow('FLUXER_CACHE_PURGE_HTTP_TOKEN is required');+ });+test('rejects a cache purge endpoint that is not an absolute http URL', async () => {for (const endpoint of ['/purge', 'purge.internal/purge', 'ftp://purge.internal/purge']) {stubMinimalEnv({FLUXER_CACHE_PURGE_ADAPTER: 'http', FLUXER_CACHE_PURGE_HTTP_ENDPOINT: endpoint});@@ -596,6 +604,7 @@ describe('ConfigLoader', () => {stubMinimalEnv({FLUXER_CACHE_PURGE_ADAPTER: 'http',FLUXER_CACHE_PURGE_HTTP_ENDPOINT: 'https://purge.internal/purge',+ FLUXER_CACHE_PURGE_HTTP_TOKEN: 'purge-token',FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS: timeout,});await expect(loadConfig()).rejects.toThrow(