Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

refactor(api): purge cache by canonical media prefix (#2802)

Hampus7412ec3395b8
diff --git a/fluxer_api/pkgs/kv_client/src/IKVProvider.ts b/fluxer_api/pkgs/kv_client/src/IKVProvider.ts
index d54946c14..82e774165 100644
--- a/fluxer_api/pkgs/kv_client/src/IKVProvider.ts
+++ b/fluxer_api/pkgs/kv_client/src/IKVProvider.ts
@@ -29,7 +29,7 @@ export interface IKVSubscription {
}
export interface KVPurgeBatchResult {
- urls: Array<string>;
+ entries: Array<string>;
tokensConsumed: number;
}
diff --git a/fluxer_api/pkgs/kv_client/src/KVClient.ts b/fluxer_api/pkgs/kv_client/src/KVClient.ts
index 4e3b3ad96..8203e7b88 100644
--- a/fluxer_api/pkgs/kv_client/src/KVClient.ts
+++ b/fluxer_api/pkgs/kv_client/src/KVClient.ts
@@ -217,7 +217,7 @@ local refillIntervalMs = tonumber(ARGV[5])
local queueSize = redis.call('SCARD', queueKey)
if queueSize == 0 then
- return '{"urls":[],"tokens":0}'
+ return '{"entries":[],"tokens":0}'
end
local tokens = maxTokens
@@ -237,14 +237,14 @@ end
local toPop = math.min(maxItems, math.floor(tokens), queueSize)
if toPop <= 0 then
redis.call('SET', bucketKey, cjson.encode({tokens = tokens, lastRefill = lastRefill}), 'EX', 3600)
- return '{"urls":[],"tokens":0}'
+ return '{"entries":[],"tokens":0}'
end
-local urls = redis.call('SPOP', queueKey, toPop)
-tokens = tokens - #urls
+local entries = redis.call('SPOP', queueKey, toPop)
+tokens = tokens - #entries
redis.call('SET', bucketKey, cjson.encode({tokens = tokens, lastRefill = lastRefill}), 'EX', 3600)
-return cjson.encode({urls = urls, tokens = #urls})
+return cjson.encode({entries = entries, tokens = #entries})
`;
const CLAIM_BULK_DELETION_SCRIPT = `
local score = redis.call('ZSCORE', KEYS[1], ARGV[1])
@@ -902,17 +902,17 @@ function parseRateLimitResult(value: unknown): KVRateLimitResult {
function parsePurgeBatchResult(value: unknown, maxItems: number): KVPurgeBatchResult {
const command = 'dequeuePurgeBatch';
if (!isJsonObject(value)) throw createInvalidResponseError(command, 'a purge batch object');
- const {urls, tokens} = value;
+ const {entries, tokens} = value;
if (
- !Array.isArray(urls) ||
- !urls.every((url): url is string => typeof url === 'string') ||
+ !Array.isArray(entries) ||
+ !entries.every((entry): entry is string => typeof entry === 'string') ||
!isNonNegativeSafeInteger(tokens) ||
- tokens !== urls.length ||
- urls.length > maxItems
+ tokens !== entries.length ||
+ entries.length > maxItems
) {
throw createInvalidResponseError(command, 'a bounded string array and matching token count');
}
- return {urls, tokensConsumed: tokens};
+ return {entries, tokensConsumed: tokens};
}
function isJsonObject(value: unknown): value is Record<string, unknown> {
diff --git a/fluxer_api/pkgs/kv_client/src/__tests__/KVClient.test.ts b/fluxer_api/pkgs/kv_client/src/__tests__/KVClient.test.ts
index 8ddbb46c7..9f3b81116 100644
--- a/fluxer_api/pkgs/kv_client/src/__tests__/KVClient.test.ts
+++ b/fluxer_api/pkgs/kv_client/src/__tests__/KVClient.test.ts
@@ -153,7 +153,7 @@ describe('KVClient script execution', () => {
},
{
name: 'dequeuePurgeBatch',
- reply: JSON.stringify({urls: ['https://fluxer.test/a.png'], tokens: 1}),
+ reply: JSON.stringify({entries: ['/attachments/1/2/a'], tokens: 1}),
keyCount: 2,
run: async (client) => client.dequeuePurgeBatch('queue:key', 'bucket:key', 10, 10, 1, 1000),
},
diff --git a/fluxer_api/src/api/infrastructure/CachePurgeAdapter.ts b/fluxer_api/src/api/infrastructure/CachePurgeAdapter.ts
index 71b96dd40..7e881da59 100644
--- a/fluxer_api/src/api/infrastructure/CachePurgeAdapter.ts
+++ b/fluxer_api/src/api/infrastructure/CachePurgeAdapter.ts
@@ -5,18 +5,13 @@ import {createHttpCachePurgeAdapter} from '@app/api/infrastructure/HttpCachePurg
import {createNoneCachePurgeAdapter} from '@app/api/infrastructure/NoneCachePurgeAdapter';
import type {CachePurgeAdapterName} from '@fluxer/config/src/MasterConfig';
-export interface CachePurgeBatch {
- readonly exact: ReadonlyArray<string>;
- readonly prefix: ReadonlyArray<string>;
-}
-
export type CachePurgeOutcome =
| {readonly kind: 'purged'}
- | {readonly kind: 'invalid_entries'; readonly status: number}
+ | {readonly kind: 'rejected'; readonly status: number}
| {readonly kind: 'failed'; readonly status: number | null; readonly error: unknown};
export interface CachePurgeAdapter {
- purge(batch: CachePurgeBatch): Promise<CachePurgeOutcome>;
+ purge(prefixes: ReadonlyArray<string>): Promise<CachePurgeOutcome>;
}
const CACHE_PURGE_ADAPTERS = {
diff --git a/fluxer_api/src/api/infrastructure/CachePurgePaths.test.ts b/fluxer_api/src/api/infrastructure/CachePurgePaths.test.ts
new file mode 100644
index 000000000..e84eaa8c6
--- /dev/null
+++ b/fluxer_api/src/api/infrastructure/CachePurgePaths.test.ts
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {Config} from '@app/api/Config';
+import {canonicalizePurgeUrl} from '@app/api/infrastructure/CachePurgePaths';
+import {afterEach, beforeEach, describe, expect, it} from 'vitest';
+
+const MEDIA = 'https://media.test';
+
+describe('canonicalizePurgeUrl', () => {
+ let previousMedia: string;
+
+ beforeEach(() => {
+ previousMedia = Config.endpoints.media;
+ Config.endpoints.media = MEDIA;
+ });
+
+ afterEach(() => {
+ Config.endpoints.media = previousMedia;
+ });
+
+ it('qualifies every prefix with the media host and no scheme', () => {
+ expect(canonicalizePurgeUrl(`${MEDIA}/attachments/1/2/photo.png`)).toEqual(['media.test/attachments/1/2/photo']);
+ });
+
+ it('drops the file extension so every served format is covered', () => {
+ expect(canonicalizePurgeUrl(`${MEDIA}/emojis/1531309058777157632.webp`)).toEqual([
+ 'media.test/emojis/1531309058777157632',
+ ]);
+ });
+
+ it('pairs an asset hash with its animated spelling in both directions', () => {
+ expect(canonicalizePurgeUrl(`${MEDIA}/avatars/1/a_b35cc3d3`)).toEqual([
+ 'media.test/avatars/1/a_b35cc3d3',
+ 'media.test/avatars/1/b35cc3d3',
+ ]);
+ expect(canonicalizePurgeUrl(`${MEDIA}/icons/1/808b11fa.webp`)).toEqual([
+ 'media.test/icons/1/808b11fa',
+ 'media.test/icons/1/a_808b11fa',
+ ]);
+ });
+
+ it('leaves a non-hash stem alone', () => {
+ expect(canonicalizePurgeUrl(`${MEDIA}/attachments/1/2/holiday.jpg`)).toEqual([
+ 'media.test/attachments/1/2/holiday',
+ ]);
+ });
+
+ it('drops the query and fragment and decodes the path', () => {
+ expect(canonicalizePurgeUrl(`${MEDIA}/attachments/1/2/ação.png?size=128#x`)).toEqual([
+ 'media.test/attachments/1/2/ação',
+ ]);
+ });
+
+ it('keeps a base path when the media endpoint carries one', () => {
+ Config.endpoints.media = `${MEDIA}/media`;
+ expect(canonicalizePurgeUrl(`${MEDIA}/media/avatars/1/b35cc3d3`)).toEqual([
+ 'media.test/media/avatars/1/b35cc3d3',
+ 'media.test/media/avatars/1/a_b35cc3d3',
+ ]);
+ expect(canonicalizePurgeUrl(`${MEDIA}/avatars/1/b35cc3d3`)).toEqual([]);
+ });
+
+ it('refuses anything that is not a media CDN object', () => {
+ expect(canonicalizePurgeUrl('https://elsewhere.test/avatars/1/b35cc3d3')).toEqual([]);
+ expect(canonicalizePurgeUrl(`${MEDIA}/avatars`)).toEqual([]);
+ expect(canonicalizePurgeUrl(`${MEDIA}/`)).toEqual([]);
+ expect(canonicalizePurgeUrl(`${MEDIA}/emojis/.webp`)).toEqual([]);
+ expect(canonicalizePurgeUrl('not-a-url')).toEqual([]);
+ });
+});
diff --git a/fluxer_api/src/api/infrastructure/CachePurgePaths.ts b/fluxer_api/src/api/infrastructure/CachePurgePaths.ts
new file mode 100644
index 000000000..891a4be71
--- /dev/null
+++ b/fluxer_api/src/api/infrastructure/CachePurgePaths.ts
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {Config} from '@app/api/Config';
+
+const EXTENSION_PATTERN = /\.[A-Za-z0-9]{1,8}$/u;
+const ASSET_HASH_PATTERN = /^[0-9a-f]{8}$/u;
+const TRAILING_SLASHES_PATTERN = /\/+$/u;
+const ANIMATED_PREFIX = 'a_';
+const MIN_PATH_SEGMENTS = 2;
+
+function mediaBase(): URL | null {
+ return URL.parse(Config.endpoints.media);
+}
+
+function decodePathname(pathname: string): string {
+ try {
+ return decodeURIComponent(pathname);
+ } catch {
+ return pathname;
+ }
+}
+
+function animationVariants(stem: string): Array<string> {
+ if (stem.startsWith(ANIMATED_PREFIX)) {
+ const bare = stem.slice(ANIMATED_PREFIX.length);
+ return ASSET_HASH_PATTERN.test(bare) ? [stem, bare] : [stem];
+ }
+ return ASSET_HASH_PATTERN.test(stem) ? [stem, `${ANIMATED_PREFIX}${stem}`] : [stem];
+}
+
+export function canonicalizePurgeUrl(url: string): Array<string> {
+ const base = mediaBase();
+ const parsed = URL.parse(url);
+ if (base === null || parsed === null || parsed.origin !== base.origin) {
+ return [];
+ }
+ const basePath = base.pathname.replace(TRAILING_SLASHES_PATTERN, '');
+ if (!parsed.pathname.startsWith(`${basePath}/`)) {
+ return [];
+ }
+ const segments = decodePathname(parsed.pathname.slice(basePath.length))
+ .split('/')
+ .filter((segment) => segment !== '');
+ if (segments.length < MIN_PATH_SEGMENTS) {
+ return [];
+ }
+ const stem = segments[segments.length - 1]!.replace(EXTENSION_PATTERN, '');
+ if (stem === '') {
+ return [];
+ }
+ const directory = segments.slice(0, -1).join('/');
+ return animationVariants(stem).map((variant) => `${base.host}${basePath}/${directory}/${variant}`);
+}
diff --git a/fluxer_api/src/api/infrastructure/CachePurgeQueue.ts b/fluxer_api/src/api/infrastructure/CachePurgeQueue.ts
index cdffe0b76..42a286d72 100644
--- a/fluxer_api/src/api/infrastructure/CachePurgeQueue.ts
+++ b/fluxer_api/src/api/infrastructure/CachePurgeQueue.ts
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-import type {CachePurgeBatch} from '@app/api/infrastructure/CachePurgeAdapter';
+import {canonicalizePurgeUrl} from '@app/api/infrastructure/CachePurgePaths';
import {Logger} from '@app/api/Logger';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
@@ -8,112 +8,69 @@ export interface IPurgeQueue {
addUrls(urls: Array<string>): Promise<void>;
}
-const EXACT_QUEUE_KEY = 'cache_purge:exact';
-const PREFIX_QUEUE_KEY = 'cache_purge:prefix';
-const EXACT_BUCKET_KEY = 'cache_purge:budget:exact';
-const PREFIX_BUCKET_KEY = 'cache_purge:budget:prefix';
+const QUEUE_KEY = 'cache_purge:queue';
+const BUDGET_KEY = 'cache_purge:budget';
const REJECTED_KEY = 'cache_purge:rejected';
-const EXACT_MAX_TOKENS = 120;
-const EXACT_REFILL_RATE = 5;
-const EXACT_REFILL_INTERVAL_MS = 1000;
-const PREFIX_MAX_TOKENS = 20;
-const PREFIX_REFILL_RATE = 1;
-const PREFIX_REFILL_INTERVAL_MS = 2000;
-
-function isPrefix(url: string): boolean {
- return url.endsWith('*') || url.endsWith('/');
-}
+const MAX_TOKENS = 120;
+const REFILL_RATE = 5;
+const REFILL_INTERVAL_MS = 1000;
export class CachePurgeQueue implements IPurgeQueue {
- private readonly kvClient: IKVProvider;
-
- constructor(kvClient: IKVProvider) {
- this.kvClient = kvClient;
- }
+ constructor(private readonly kvClient: IKVProvider) {}
async addUrls(urls: Array<string>): Promise<void> {
- if (urls.length === 0) {
- return;
- }
- const exactUrls: Array<string> = [];
- const prefixUrls: Array<string> = [];
+ const prefixes = new Set<string>();
for (const url of urls) {
- const trimmed = url.trim();
- if (trimmed === '') {
+ const canonical = canonicalizePurgeUrl(url);
+ if (canonical.length === 0) {
+ Logger.warn({url}, 'Skipped a cache purge URL that is not a media CDN object');
continue;
}
- if (isPrefix(trimmed)) {
- prefixUrls.push(trimmed);
- } else {
- exactUrls.push(trimmed);
+ for (const prefix of canonical) {
+ prefixes.add(prefix);
}
}
+ if (prefixes.size === 0) {
+ return;
+ }
try {
- await this.addToSets({exact: exactUrls, prefix: prefixUrls});
- Logger.debug({exact: exactUrls.length, prefix: prefixUrls.length}, 'Added URLs to cache purge queue');
+ await this.kvClient.sadd(QUEUE_KEY, ...prefixes);
+ Logger.debug({prefixes: prefixes.size}, 'Added prefixes to the cache purge queue');
} catch (error) {
- Logger.error(
- {error, exact: exactUrls.length, prefix: prefixUrls.length},
- 'Failed to add URLs to cache purge queue',
- );
+ Logger.error({error, prefixes: prefixes.size}, 'Failed to add prefixes to the cache purge queue');
throw error;
}
}
- async dequeueBatch(): Promise<CachePurgeBatch> {
- const exact = await this.kvClient.dequeuePurgeBatch(
- EXACT_QUEUE_KEY,
- EXACT_BUCKET_KEY,
- EXACT_MAX_TOKENS,
- EXACT_MAX_TOKENS,
- EXACT_REFILL_RATE,
- EXACT_REFILL_INTERVAL_MS,
+ async dequeueBatch(): Promise<Array<string>> {
+ const batch = await this.kvClient.dequeuePurgeBatch(
+ QUEUE_KEY,
+ BUDGET_KEY,
+ MAX_TOKENS,
+ MAX_TOKENS,
+ REFILL_RATE,
+ REFILL_INTERVAL_MS,
);
- try {
- const prefix = await this.kvClient.dequeuePurgeBatch(
- PREFIX_QUEUE_KEY,
- PREFIX_BUCKET_KEY,
- PREFIX_MAX_TOKENS,
- PREFIX_MAX_TOKENS,
- PREFIX_REFILL_RATE,
- PREFIX_REFILL_INTERVAL_MS,
- );
- return {exact: exact.urls, prefix: prefix.urls};
- } catch (error) {
- await this.requeue({exact: exact.urls, prefix: []});
- throw error;
- }
+ return batch.entries;
}
- async requeue(batch: CachePurgeBatch): Promise<void> {
+ async requeue(prefixes: ReadonlyArray<string>): Promise<void> {
+ if (prefixes.length === 0) {
+ return;
+ }
try {
- await this.addToSets(batch);
+ await this.kvClient.sadd(QUEUE_KEY, ...prefixes);
} catch (error) {
- Logger.error(
- {error, exact: batch.exact.length, prefix: batch.prefix.length},
- 'Failed to requeue cache purge entries',
- );
+ Logger.error({error, prefixes: prefixes.length}, 'Failed to requeue cache purge prefixes');
throw error;
}
}
- async reject(batch: CachePurgeBatch): Promise<void> {
- await this.kvClient.sadd(REJECTED_KEY, ...batch.exact, ...batch.prefix);
- Logger.error(
- {exact: batch.exact.length, prefix: batch.prefix.length, key: REJECTED_KEY},
- 'Set aside cache purge entries the endpoint rejected',
- );
- }
-
- private async addToSets(batch: CachePurgeBatch): Promise<void> {
- const ops: Array<Promise<number>> = [];
- if (batch.exact.length > 0) {
- ops.push(this.kvClient.sadd(EXACT_QUEUE_KEY, ...batch.exact));
- }
- if (batch.prefix.length > 0) {
- ops.push(this.kvClient.sadd(PREFIX_QUEUE_KEY, ...batch.prefix));
+ async reject(prefixes: ReadonlyArray<string>): Promise<void> {
+ if (prefixes.length === 0) {
+ return;
}
- await Promise.all(ops);
+ await this.kvClient.sadd(REJECTED_KEY, ...prefixes);
}
}
diff --git a/fluxer_api/src/api/infrastructure/HttpCachePurgeAdapter.ts b/fluxer_api/src/api/infrastructure/HttpCachePurgeAdapter.ts
index 507821922..2af8c5c08 100644
--- a/fluxer_api/src/api/infrastructure/HttpCachePurgeAdapter.ts
+++ b/fluxer_api/src/api/infrastructure/HttpCachePurgeAdapter.ts
@@ -1,26 +1,18 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {APICachePurgeConfig} from '@app/api/config/APIConfig';
-import type {CachePurgeAdapter, CachePurgeBatch, CachePurgeOutcome} from '@app/api/infrastructure/CachePurgeAdapter';
+import type {CachePurgeAdapter, CachePurgeOutcome} from '@app/api/infrastructure/CachePurgeAdapter';
import * as FetchUtils from '@app/api/utils/FetchUtils';
-function stripOneTrailingAsterisk(entry: string): string {
- return entry.endsWith('*') ? entry.slice(0, -1) : entry;
-}
-
export function createHttpCachePurgeAdapter(config: APICachePurgeConfig): CachePurgeAdapter {
return {
- async purge(batch: CachePurgeBatch): Promise<CachePurgeOutcome> {
- const headers: Record<string, string> = {'Content-Type': 'application/json'};
- if (config.http.token !== '') {
- headers.Authorization = `Bearer ${config.http.token}`;
- }
+ async purge(prefixes: ReadonlyArray<string>): Promise<CachePurgeOutcome> {
let response: Response;
try {
response = await fetch(config.http.endpoint, {
method: 'POST',
- headers,
- body: JSON.stringify({exact: batch.exact, prefix: batch.prefix.map(stripOneTrailingAsterisk)}),
+ headers: {'Content-Type': 'application/json', Authorization: `Bearer ${config.http.token}`},
+ body: JSON.stringify({prefixes}),
redirect: 'manual',
signal: AbortSignal.timeout(config.http.timeoutMs),
});
@@ -32,7 +24,7 @@ export function createHttpCachePurgeAdapter(config: APICachePurgeConfig): CacheP
return {kind: 'purged'};
}
if (response.status === 400 || response.status === 422) {
- return {kind: 'invalid_entries', status: response.status};
+ return {kind: 'rejected', status: response.status};
}
return {kind: 'failed', status: response.status, error: null};
},
diff --git a/fluxer_api/src/api/test/mocks/MockKVProvider.ts b/fluxer_api/src/api/test/mocks/MockKVProvider.ts
index 64f5e22c9..18c2b6ce4 100644
--- a/fluxer_api/src/api/test/mocks/MockKVProvider.ts
+++ b/fluxer_api/src/api/test/mocks/MockKVProvider.ts
@@ -704,7 +704,7 @@ export class MockKVProvider implements IKVProvider {
refillRate: number,
refillIntervalMs: number,
): Promise<{
- urls: Array<string>;
+ entries: Array<string>;
tokensConsumed: number;
}> {
this.dequeuePurgeBatchSpy(queueKey, bucketKey, maxItems, maxTokens, refillRate, refillIntervalMs);
@@ -734,13 +734,13 @@ export class MockKVProvider implements IKVProvider {
if (toPop <= 0) {
this.stringStore.set(bucketKey, JSON.stringify({tokens, lastRefill}));
this.expiries.set(bucketKey, now + 3600 * 1000);
- return {urls: [], tokensConsumed: 0};
+ return {entries: [], tokensConsumed: 0};
}
- const urls = await this.spop(queueKey, toPop);
- tokens -= urls.length;
+ const entries = await this.spop(queueKey, toPop);
+ tokens -= entries.length;
this.stringStore.set(bucketKey, JSON.stringify({tokens, lastRefill}));
this.expiries.set(bucketKey, now + 3600 * 1000);
- return {urls, tokensConsumed: urls.length};
+ return {entries, tokensConsumed: entries.length};
}
async evalScript(
diff --git a/fluxer_api/src/api/worker/tasks/ProcessCachePurgeQueue.ts b/fluxer_api/src/api/worker/tasks/ProcessCachePurgeQueue.ts
index 94880b951..670a5a45b 100644
--- a/fluxer_api/src/api/worker/tasks/ProcessCachePurgeQueue.ts
+++ b/fluxer_api/src/api/worker/tasks/ProcessCachePurgeQueue.ts
@@ -1,104 +1,53 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Config} from '@app/api/Config';
-import {
- type CachePurgeBatch,
- type CachePurgeOutcome,
- createCachePurgeAdapter,
-} from '@app/api/infrastructure/CachePurgeAdapter';
+import {type CachePurgeOutcome, createCachePurgeAdapter} from '@app/api/infrastructure/CachePurgeAdapter';
import {CachePurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import {Logger} from '@app/api/Logger';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
-import type {CachePurgeAdapterName} from '@fluxer/config/src/MasterConfig';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
-const RUN_DEADLINE_MS = 10_000;
-
type CachePurgeFailure = Extract<CachePurgeOutcome, {kind: 'failed'}>;
-function describeBatch(adapter: CachePurgeAdapterName, batch: CachePurgeBatch) {
- return {adapter, exact: batch.exact.length, prefix: batch.prefix.length};
-}
-
-function logFailure(adapter: CachePurgeAdapterName, requeued: CachePurgeBatch, failure: CachePurgeFailure): void {
+function logFailure(context: Record<string, unknown>, failure: CachePurgeFailure): void {
const {status, error} = failure;
- const context = {...describeBatch(adapter, requeued), status, error};
if (status === null || status === 408 || status === 429 || status >= 500) {
- Logger.warn(context, 'Cache purge request failed, requeued its entries');
+ Logger.warn({...context, status, error}, 'Cache purge request failed, requeued its prefixes');
return;
}
- Logger.error(context, 'Cache purge endpoint refused the request, requeued its entries');
-}
-
-function untriedFrom(singles: ReadonlyArray<CachePurgeBatch>, index: number): CachePurgeBatch {
- const untried = singles.slice(index);
- return {exact: untried.flatMap((single) => single.exact), prefix: untried.flatMap((single) => single.prefix)};
+ Logger.error({...context, status, error}, 'Cache purge endpoint refused the request, requeued its prefixes');
}
const processCachePurgeQueue: WorkerTaskHandler = async (_payload, _helpers) => {
- const adapterName = Config.cachePurge.adapter;
- if (adapterName === 'none') {
+ const adapter = Config.cachePurge.adapter;
+ if (adapter === 'none') {
Logger.warn('Skipped a cache purge run because the adapter is none');
return;
}
- const adapter = createCachePurgeAdapter(Config.cachePurge);
const queue = new CachePurgeQueue(getWorkerDependencies().kvClient);
- const startedAt = Date.now();
- const batch = await queue.dequeueBatch();
- if (batch.exact.length === 0 && batch.prefix.length === 0) {
+ const prefixes = await queue.dequeueBatch();
+ if (prefixes.length === 0) {
return;
}
+ const context = {adapter, prefixes: prefixes.length};
let outcome: CachePurgeOutcome;
try {
- outcome = await adapter.purge(batch);
+ outcome = await createCachePurgeAdapter(Config.cachePurge).purge(prefixes);
} catch (error) {
- await queue.requeue(batch);
+ await queue.requeue(prefixes);
throw error;
}
if (outcome.kind === 'purged') {
- Logger.debug(describeBatch(adapterName, batch), 'Purged a batch of cache entries');
+ Logger.debug(context, 'Purged a batch of cache prefixes');
return;
}
- if (outcome.kind === 'failed') {
- await queue.requeue(batch);
- logFailure(adapterName, batch, outcome);
+ if (outcome.kind === 'rejected') {
+ await queue.reject(prefixes);
+ Logger.error({...context, status: outcome.status}, 'Cache purge endpoint rejected the batch, set it aside');
return;
}
- Logger.warn(
- {...describeBatch(adapterName, batch), status: outcome.status},
- 'Cache purge endpoint rejected the batch, retrying each entry on its own',
- );
- const singles: Array<CachePurgeBatch> = [
- ...batch.exact.map((entry) => ({exact: [entry], prefix: []})),
- ...batch.prefix.map((entry) => ({exact: [], prefix: [entry]})),
- ];
- for (const [index, single] of singles.entries()) {
- if (Date.now() - startedAt >= RUN_DEADLINE_MS) {
- const untried = untriedFrom(singles, index);
- await queue.requeue(untried);
- Logger.warn(
- describeBatch(adapterName, untried),
- 'Cache purge run reached its deadline, requeued untried entries',
- );
- return;
- }
- let singleOutcome: CachePurgeOutcome;
- try {
- singleOutcome = await adapter.purge(single);
- if (singleOutcome.kind === 'invalid_entries') {
- await queue.reject(single);
- }
- } catch (error) {
- await queue.requeue(untriedFrom(singles, index));
- throw error;
- }
- if (singleOutcome.kind === 'failed') {
- const untried = untriedFrom(singles, index);
- await queue.requeue(untried);
- logFailure(adapterName, untried, singleOutcome);
- return;
- }
- }
+ await queue.requeue(prefixes);
+ logFailure(context, outcome);
};
export default processCachePurgeQueue;
diff --git a/fluxer_api/src/api/worker/tests/ProcessCachePurgeQueue.test.ts b/fluxer_api/src/api/worker/tests/ProcessCachePurgeQueue.test.ts
index b628de50c..e9acf2c64 100644
--- a/fluxer_api/src/api/worker/tests/ProcessCachePurgeQueue.test.ts
+++ b/fluxer_api/src/api/worker/tests/ProcessCachePurgeQueue.test.ts
@@ -13,15 +13,14 @@ import {delay, HttpResponse, http} from 'msw';
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
const HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
-const ENDPOINT = 'https://cache-purge.test/purge';
+const ENDPOINT = 'https://cache-purge.test/__cache/purge';
const MEDIA = 'https://media.test';
-const EXACT_KEY = 'cache_purge:exact';
-const PREFIX_KEY = 'cache_purge:prefix';
+const QUEUE_KEY = 'cache_purge:queue';
+const BUDGET_KEY = 'cache_purge:budget';
const REJECTED_KEY = 'cache_purge:rejected';
interface PurgeBody {
- exact: Array<string>;
- prefix: Array<string>;
+ prefixes: Array<string>;
}
interface RecordedRequest {
@@ -52,10 +51,6 @@ function recordPurges(respond: (body: PurgeBody) => Response | Promise<Response>
return requests;
}
-function entryCount(body: PurgeBody): number {
- return body.exact.length + body.prefix.length;
-}
-
async function members(kvClient: MockKVProvider, key: string): Promise<Array<string>> {
return (await kvClient.smembers(key)).sort();
}
@@ -66,9 +61,12 @@ function sorted(values: Array<string>): Array<string> {
describe('processCachePurgeQueue', () => {
let previousCachePurge: APICachePurgeConfig;
+ let previousMedia: string;
beforeEach(() => {
previousCachePurge = {adapter: Config.cachePurge.adapter, http: Config.cachePurge.http};
+ previousMedia = Config.endpoints.media;
+ Config.endpoints.media = MEDIA;
Config.cachePurge.adapter = 'http';
Config.cachePurge.http = {endpoint: ENDPOINT, token: 'test-token', timeoutMs: 50};
});
@@ -76,15 +74,18 @@ describe('processCachePurgeQueue', () => {
afterEach(() => {
Config.cachePurge.adapter = previousCachePurge.adapter;
Config.cachePurge.http = previousCachePurge.http;
+ Config.endpoints.media = previousMedia;
clearWorkerDependencies();
vi.useRealTimers();
});
- it('sends queued exact and prefix entries in one request and drains the queue', async () => {
+ it('sends host qualified prefixes in one request and drains the queue', async () => {
const {kvClient, queue} = createHarness();
- const exact = [`${MEDIA}/avatars/1/a_abc`, `${MEDIA}/emojis/9.webp`, `${MEDIA}/attachments/1/2/ação.png`];
- const prefix = [`${MEDIA}/some/dir/`];
- await queue.addUrls([...exact, ...prefix]);
+ await queue.addUrls([
+ `${MEDIA}/avatars/1/a_b35cc3d3`,
+ `${MEDIA}/emojis/9.webp`,
+ `${MEDIA}/attachments/1/2/ação.png`,
+ ]);
const requests = recordPurges(() => new HttpResponse(null, {status: 204}));
await processCachePurgeQueue({}, HELPERS);
@@ -92,55 +93,69 @@ describe('processCachePurgeQueue', () => {
expect(requests).toHaveLength(1);
expect(requests[0]!.authorization).toBe('Bearer test-token');
expect(requests[0]!.contentType).toBe('application/json');
- expect(sorted(requests[0]!.body.exact)).toEqual(sorted(exact));
- expect(requests[0]!.body.prefix).toEqual(prefix);
- expect(await members(kvClient, EXACT_KEY)).toEqual([]);
- expect(await members(kvClient, PREFIX_KEY)).toEqual([]);
+ expect(sorted(requests[0]!.body.prefixes)).toEqual(
+ sorted([
+ 'media.test/avatars/1/a_b35cc3d3',
+ 'media.test/avatars/1/b35cc3d3',
+ 'media.test/emojis/9',
+ 'media.test/attachments/1/2/ação',
+ ]),
+ );
+ expect(await members(kvClient, QUEUE_KEY)).toEqual([]);
});
- it('strips a trailing asterisk from a prefix entry and keeps a trailing slash', async () => {
- const {kvClient, queue} = createHarness();
- await queue.addUrls([`${MEDIA}/stickers/1**`, `${MEDIA}/emojis/`]);
+ it('collapses every extension of one asset into a single prefix', async () => {
+ const {queue} = createHarness();
+ await queue.addUrls([`${MEDIA}/stickers/7.webp`, `${MEDIA}/stickers/7.gif`, `${MEDIA}/stickers/7.png`]);
const requests = recordPurges(() => new HttpResponse(null, {status: 204}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
- expect(requests[0]!.body.exact).toEqual([]);
- expect(sorted(requests[0]!.body.prefix)).toEqual([`${MEDIA}/emojis/`, `${MEDIA}/stickers/1*`]);
- expect(await members(kvClient, PREFIX_KEY)).toEqual([]);
+ expect(requests[0]!.body.prefixes).toEqual(['media.test/stickers/7']);
});
- it('sends no authorization header when no token is configured', async () => {
- Config.cachePurge.http.token = '';
- const {queue} = createHarness();
- await queue.addUrls([`${MEDIA}/avatars/1/abc`]);
- const requests = recordPurges(() => new HttpResponse(null, {status: 204}));
+ it.each([200, 202, 204])('treats %i as accepted and drops the batch', async (status) => {
+ const {kvClient, queue} = createHarness();
+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);
+ const requests = recordPurges(() => new HttpResponse(null, {status}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
- expect(requests[0]!.authorization).toBeNull();
+ expect(await members(kvClient, QUEUE_KEY)).toEqual([]);
+ expect(await members(kvClient, REJECTED_KEY)).toEqual([]);
+ });
+
+ it('never queues a URL that is not a media CDN object', async () => {
+ const {kvClient, queue} = createHarness();
+ await queue.addUrls(['https://elsewhere.test/avatars/1/b35cc3d3', `${MEDIA}/avatars`, 'not-a-url']);
+ const requests = recordPurges(() => new HttpResponse(null, {status: 204}));
+
+ await processCachePurgeQueue({}, HELPERS);
+
+ expect(requests).toEqual([]);
+ expect(await members(kvClient, QUEUE_KEY)).toEqual([]);
});
it('requeues the whole batch after a server error', async () => {
const {kvClient, queue} = createHarness();
- const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];
- const prefix = [`${MEDIA}/some/dir/`];
- await queue.addUrls([...exact, ...prefix]);
+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`, `${MEDIA}/attachments/1/3/b.png`]);
const requests = recordPurges(() => new HttpResponse(null, {status: 503}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));
- expect(await members(kvClient, PREFIX_KEY)).toEqual(prefix);
+ expect(await members(kvClient, QUEUE_KEY)).toEqual([
+ 'media.test/attachments/1/2/a',
+ 'media.test/attachments/1/3/b',
+ ]);
+ expect(await members(kvClient, REJECTED_KEY)).toEqual([]);
});
it('requeues the batch when the endpoint does not answer in time', async () => {
const {kvClient, queue} = createHarness();
- const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];
- await queue.addUrls(exact);
+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);
const requests = recordPurges(async () => {
await delay('infinite');
return new HttpResponse(null, {status: 204});
@@ -149,25 +164,23 @@ describe('processCachePurgeQueue', () => {
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));
+ expect(await members(kvClient, QUEUE_KEY)).toEqual(['media.test/attachments/1/2/a']);
});
it('requeues the batch when the endpoint is unreachable', async () => {
const {kvClient, queue} = createHarness();
- const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];
- await queue.addUrls(exact);
+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);
const requests = recordPurges(() => HttpResponse.error());
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));
+ expect(await members(kvClient, QUEUE_KEY)).toEqual(['media.test/attachments/1/2/a']);
});
it('requeues the batch on a redirect without following it', async () => {
const {kvClient, queue} = createHarness();
- const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`];
- await queue.addUrls(exact);
+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);
const redirectTarget = 'https://cache-purge.test/elsewhere';
const redirectedRequests: Array<string> = [];
server.use(
@@ -182,122 +195,51 @@ describe('processCachePurgeQueue', () => {
expect(requests).toHaveLength(1);
expect(redirectedRequests).toEqual([]);
- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));
+ expect(await members(kvClient, QUEUE_KEY)).toEqual(['media.test/attachments/1/2/a']);
});
- it('requeues the batch on an authorisation failure without splitting it', async () => {
+ it('requeues rather than discards when the bearer token is refused', async () => {
const {kvClient, queue} = createHarness();
- const exact = [`${MEDIA}/avatars/1/abc`, `${MEDIA}/banners/1/def`, `${MEDIA}/emojis/9.webp`];
- await queue.addUrls(exact);
+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);
const requests = recordPurges(() => new HttpResponse(null, {status: 401}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toHaveLength(1);
- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted(exact));
+ expect(await members(kvClient, QUEUE_KEY)).toEqual(['media.test/attachments/1/2/a']);
expect(await members(kvClient, REJECTED_KEY)).toEqual([]);
});
- it('sets aside only the entry the endpoint rejects on its own', async () => {
+ it.each([400, 422])('sets the batch aside when the endpoint answers %i', async (status) => {
const {kvClient, queue} = createHarness();
- const poison = `${MEDIA}/attachments/1/2/poison.png`;
- await queue.addUrls([`${MEDIA}/avatars/1/abc`, poison, `${MEDIA}/emojis/9.webp`]);
- const requests = recordPurges((body) =>
- body.exact.includes(poison) ? new HttpResponse(null, {status: 422}) : new HttpResponse(null, {status: 204}),
- );
+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`, `${MEDIA}/emojis/9.webp`]);
+ const requests = recordPurges(() => new HttpResponse(null, {status}));
await processCachePurgeQueue({}, HELPERS);
- expect(requests.map((request) => entryCount(request.body))).toEqual([3, 1, 1, 1]);
- expect(await members(kvClient, REJECTED_KEY)).toEqual([poison]);
- expect(await members(kvClient, EXACT_KEY)).toEqual([]);
- });
-
- it('splits the batch when the endpoint answers 400', async () => {
- const {kvClient, queue} = createHarness();
- const poison = `${MEDIA}/attachments/1/2/poison.png`;
- await queue.addUrls([`${MEDIA}/avatars/1/abc`, poison, `${MEDIA}/emojis/9.webp`]);
- const requests = recordPurges((body) =>
- body.exact.includes(poison) ? new HttpResponse(null, {status: 400}) : new HttpResponse(null, {status: 204}),
- );
-
- await processCachePurgeQueue({}, HELPERS);
-
- expect(requests.map((request) => entryCount(request.body))).toEqual([3, 1, 1, 1]);
- expect(await members(kvClient, REJECTED_KEY)).toEqual([poison]);
- expect(await members(kvClient, EXACT_KEY)).toEqual([]);
- });
-
- it('keeps untried exact and prefix entries queued when a single-entry retry fails', async () => {
- const {kvClient, queue} = createHarness();
- const first = `${MEDIA}/avatars/1/abc`;
- const second = `${MEDIA}/banners/1/def`;
- const third = `${MEDIA}/emojis/9.webp`;
- const directory = `${MEDIA}/some/dir/`;
- await queue.addUrls([first, second, third, directory]);
- let singleRequests = 0;
- const requests = recordPurges((body) => {
- if (entryCount(body) > 1) {
- return new HttpResponse(null, {status: 422});
- }
- singleRequests++;
- return new HttpResponse(null, {status: singleRequests === 1 ? 204 : 503});
- });
-
- await processCachePurgeQueue({}, HELPERS);
-
- expect(requests.map((request) => request.body.exact)).toEqual([[first, second, third], [first], [second]]);
- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted([second, third]));
- expect(await members(kvClient, PREFIX_KEY)).toEqual([directory]);
- expect(await members(kvClient, REJECTED_KEY)).toEqual([]);
- });
-
- it('requeues untried entries when the fallback runs out of time', async () => {
- vi.useFakeTimers({toFake: ['Date']});
- vi.setSystemTime(new Date('2026-09-13T00:00:00.000Z'));
- const {kvClient, queue} = createHarness();
- const entries = [1, 2, 3, 4, 5].map((index) => `${MEDIA}/avatars/${index}/abc`);
- await queue.addUrls(entries);
- const requests = recordPurges((body) => {
- if (entryCount(body) > 1) {
- return new HttpResponse(null, {status: 422});
- }
- vi.setSystemTime(Date.now() + 4_000);
- return new HttpResponse(null, {status: 204});
- });
-
- await processCachePurgeQueue({}, HELPERS);
-
- expect(requests.map((request) => request.body.exact)).toEqual([entries, [entries[0]], [entries[1]], [entries[2]]]);
- expect(await members(kvClient, EXACT_KEY)).toEqual(sorted([entries[3]!, entries[4]!]));
- expect(await members(kvClient, REJECTED_KEY)).toEqual([]);
+ expect(requests).toHaveLength(1);
+ expect(await members(kvClient, QUEUE_KEY)).toEqual([]);
+ expect(await members(kvClient, REJECTED_KEY)).toEqual(['media.test/attachments/1/2/a', 'media.test/emojis/9']);
});
it('leaves the queue untouched when the adapter is none', async () => {
Config.cachePurge.adapter = 'none';
const {kvClient, queue} = createHarness();
- const exact = [`${MEDIA}/avatars/1/abc`];
- const prefix = [`${MEDIA}/some/dir/`];
- await queue.addUrls([...exact, ...prefix]);
+ await queue.addUrls([`${MEDIA}/attachments/1/2/a.png`]);
const requests = recordPurges(() => new HttpResponse(null, {status: 204}));
await processCachePurgeQueue({}, HELPERS);
expect(requests).toEqual([]);
- expect(await members(kvClient, EXACT_KEY)).toEqual(exact);
- expect(await members(kvClient, PREFIX_KEY)).toEqual(prefix);
- expect(await kvClient.get('cache_purge:budget:exact')).toBeNull();
- expect(await kvClient.get('cache_purge:budget:prefix')).toBeNull();
+ expect(await members(kvClient, QUEUE_KEY)).toEqual(['media.test/attachments/1/2/a']);
+ expect(await kvClient.get(BUDGET_KEY)).toBeNull();
});
it('sends no more than the token bucket allows and refills at the configured rate', async () => {
vi.useFakeTimers({toFake: ['Date']});
- vi.setSystemTime(new Date('2026-09-13T00:00:00.000Z'));
+ vi.setSystemTime(new Date('2026-09-16T00:00:00.000Z'));
const {kvClient, queue} = createHarness();
- await queue.addUrls([
- ...Array.from({length: 200}, (_, index) => `${MEDIA}/avatars/${index}/abc`),
- ...Array.from({length: 30}, (_, index) => `${MEDIA}/dirs/${index}/`),
- ]);
+ await queue.addUrls(Array.from({length: 200}, (_, index) => `${MEDIA}/attachments/1/${index}/a.png`));
const requests = recordPurges(() => new HttpResponse(null, {status: 204}));
await processCachePurgeQueue({}, HELPERS);
@@ -305,11 +247,7 @@ describe('processCachePurgeQueue', () => {
vi.setSystemTime(Date.now() + 10_000);
await processCachePurgeQueue({}, HELPERS);
- expect(requests.map((request) => [request.body.exact.length, request.body.prefix.length])).toEqual([
- [120, 20],
- [50, 5],
- ]);
- expect(await kvClient.scard(EXACT_KEY)).toBe(30);
- expect(await kvClient.scard(PREFIX_KEY)).toBe(5);
+ expect(requests.map((request) => request.body.prefixes.length)).toEqual([120, 50]);
+ expect(await kvClient.scard(QUEUE_KEY)).toBe(30);
});
});
diff --git a/packages/config/src/ConfigLoader.ts b/packages/config/src/ConfigLoader.ts
index 7d173bd4d..b3e1ca9f5 100644
--- a/packages/config/src/ConfigLoader.ts
+++ b/packages/config/src/ConfigLoader.ts
@@ -484,7 +484,8 @@ function validateCachePurgeConfig(config: MasterConfig): void {
) {
throw new Error('FLUXER_CACHE_PURGE_HTTP_ENDPOINT must be an absolute http or https URL without credentials');
}
- if (!/^[\x21-\x7e]*$/u.test(cachePurge.http.token)) {
+ requireString(cachePurge.http.token, 'FLUXER_CACHE_PURGE_HTTP_TOKEN');
+ if (!/^[\x21-\x7e]+$/u.test(cachePurge.http.token)) {
throw new Error('FLUXER_CACHE_PURGE_HTTP_TOKEN must contain only visible ASCII characters');
}
assertIntegerInRange(cachePurge.http.timeout_ms, 'FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS', 1_000, 10_000);
diff --git a/packages/config/src/__tests__/ConfigLoader.test.ts b/packages/config/src/__tests__/ConfigLoader.test.ts
index 3b1eb6242..fb2798968 100644
--- a/packages/config/src/__tests__/ConfigLoader.test.ts
+++ b/packages/config/src/__tests__/ConfigLoader.test.ts
@@ -572,6 +572,14 @@ describe('ConfigLoader', () => {
await expect(loadConfig()).rejects.toThrow('FLUXER_CACHE_PURGE_HTTP_ENDPOINT is required');
});
+ test('rejects the http cache purge adapter without a token', async () => {
+ stubMinimalEnv({
+ FLUXER_CACHE_PURGE_ADAPTER: 'http',
+ FLUXER_CACHE_PURGE_HTTP_ENDPOINT: 'https://purge.internal/purge',
+ });
+ await expect(loadConfig()).rejects.toThrow('FLUXER_CACHE_PURGE_HTTP_TOKEN is required');
+ });
+
test('rejects a cache purge endpoint that is not an absolute http URL', async () => {
for (const endpoint of ['/purge', 'purge.internal/purge', 'ftp://purge.internal/purge']) {
stubMinimalEnv({FLUXER_CACHE_PURGE_ADAPTER: 'http', FLUXER_CACHE_PURGE_HTTP_ENDPOINT: endpoint});
@@ -596,6 +604,7 @@ describe('ConfigLoader', () => {
stubMinimalEnv({
FLUXER_CACHE_PURGE_ADAPTER: 'http',
FLUXER_CACHE_PURGE_HTTP_ENDPOINT: 'https://purge.internal/purge',
+ FLUXER_CACHE_PURGE_HTTP_TOKEN: 'purge-token',
FLUXER_CACHE_PURGE_HTTP_TIMEOUT_MS: timeout,
});
await expect(loadConfig()).rejects.toThrow(