The minimalistic document archiving platform.
feat(kv-store): add bulk purge of expired entries (#1120)
Corentin Thomassetdfb22ba5f403
diff --git a/apps/papra-server/src/modules/kv-store/drivers/kv-store-drivers.types.ts b/apps/papra-server/src/modules/kv-store/drivers/kv-store-drivers.types.tsindex 0ac7a587..82dac2b7 100644--- a/apps/papra-server/src/modules/kv-store/drivers/kv-store-drivers.types.ts+++ b/apps/papra-server/src/modules/kv-store/drivers/kv-store-drivers.types.ts@@ -7,6 +7,8 @@ export type KvStoreDriver = {get: (key: string) => Promise<JsonSerializableValue | undefined>;set: (key: string, value: JsonSerializableValue, options?: { ttlMs?: number }) => Promise<void>;delete: (key: string) => Promise<void>;+ // Optional bulk purge of expired entries, only implemented by lazy-delete drivers (e.g. libsql) that would otherwise accumulate expired rows.+ deleteExpired?: () => Promise<{ deletedCount: number }>;};export type KvStoreDriverFactory = (args: { config: Config; db: Database }) => KvStoreDriver;diff --git a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.repository.ts b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.repository.tsindex 710f549a..dd9f7223 100644--- a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.repository.ts+++ b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.repository.ts@@ -13,6 +13,7 @@ export function createKvStoreRepository({ db }: { db: Database }) {upsertEntry,deleteEntry,deleteExpiredEntry,+ deleteAllExpiredEntries,},{ db },);@@ -46,3 +47,17 @@ async function deleteExpiredEntry({ key, db, now = new Date() }: { key: string;),);}++async function deleteAllExpiredEntries({ db, now = new Date() }: { db: Database; now?: Date }) {+ const deleted = await db+ .delete(kvStoreTable)+ .where(+ and(+ isNotNull(kvStoreTable.expiresAt),+ lte(kvStoreTable.expiresAt, now),+ ),+ )+ .returning({ key: kvStoreTable.key });++ return { deletedCount: deleted.length };+}diff --git a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.test.ts b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.test.tsindex 0a909df2..4182f328 100644--- a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.test.ts+++ b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.test.ts@@ -1,16 +1,45 @@-import { describe } from 'vitest';+import { describe, expect, test, vi } from 'vitest';import { kvStoreMigration } from '../../../../migrations/list/0020-kv-store.migration';import { setupDatabase } from '../../../app/database/database';import { runKvStoreDriverTestSuite } from '../kv-store-drivers.test-suite';import { createLibsqlKvStoreDriver } from './libsql.kv-store-driver';+async function createDriver() {+ const { db } = setupDatabase({ url: ':memory:' });+ await kvStoreMigration.up({ db });++ return { driver: createLibsqlKvStoreDriver({ db }) };+}+describe('libsql.kv-store-driver', () => {describe('createLibsqlKvStoreDriver', () => {- runKvStoreDriverTestSuite(async () => {- const { db } = setupDatabase({ url: ':memory:' });- await kvStoreMigration.up({ db });+ runKvStoreDriverTestSuite(createDriver);++ describe('deleteExpired', () => {+ test('removes only entries whose ttl has elapsed and returns how many were deleted', async () => {+ vi.useFakeTimers();++ const { driver } = await createDriver();++ await driver.set('expired-1', 'value', { ttlMs: 1_000 });+ await driver.set('expired-2', 'value', { ttlMs: 1_000 });+ await driver.set('still-valid', 'value', { ttlMs: 10_000 });+ await driver.set('no-ttl', 'value');++ vi.advanceTimersByTime(1_001);++ expect(await driver.deleteExpired!()).to.eql({ deletedCount: 2 });++ // A second run finds nothing left to purge.+ expect(await driver.deleteExpired!()).to.eql({ deletedCount: 0 });++ expect(await driver.get('expired-1')).to.eql(undefined);+ expect(await driver.get('expired-2')).to.eql(undefined);+ expect(await driver.get('still-valid')).to.eql('value');+ expect(await driver.get('no-ttl')).to.eql('value');- return { driver: createLibsqlKvStoreDriver({ db }) };+ vi.useRealTimers();+ });});});});diff --git a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.ts b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.tsindex 055c2ba4..0f75e47c 100644--- a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.ts+++ b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.ts@@ -42,5 +42,9 @@ export function createLibsqlKvStoreDriver({ db }: { db: Database }): KvStoreDrivawait repository.deleteEntry({ key });},+ deleteExpired: async (now = new Date()) => {+ return repository.deleteAllExpiredEntries({ now });+ },+};}diff --git a/apps/papra-server/src/modules/kv-store/kv-store.ts b/apps/papra-server/src/modules/kv-store/kv-store.tsindex fed91f3c..24eb56bd 100644--- a/apps/papra-server/src/modules/kv-store/kv-store.ts+++ b/apps/papra-server/src/modules/kv-store/kv-store.ts@@ -17,6 +17,8 @@ export function createKvStore({logger?: Logger;}): KvStore {return {+ ...(driver.deleteExpired ? { purgeExpired: driver.deleteExpired } : {}),+defineScope: ({ prefix, schema, defaultTtlMs }) => {const buildKey = (key: string) => joinKeyParts([prefix, key]);diff --git a/apps/papra-server/src/modules/kv-store/kv-store.types.ts b/apps/papra-server/src/modules/kv-store/kv-store.types.tsindex 67cbea11..162623ba 100644--- a/apps/papra-server/src/modules/kv-store/kv-store.types.ts+++ b/apps/papra-server/src/modules/kv-store/kv-store.types.ts@@ -16,4 +16,6 @@ export type KvStore = {schema: TSchema;defaultTtlMs?: number;}) => KvStoreScope<TSchema>;+ // Only present for lazy-delete drivers (e.g. libsql) that accumulate expired entries; eager drivers (e.g. in-memory) omit it.+ purgeExpired?: () => Promise<{ deletedCount: number }>;};diff --git a/apps/papra-server/src/modules/kv-store/tasks/purge-expired-kv-entries.task.ts b/apps/papra-server/src/modules/kv-store/tasks/purge-expired-kv-entries.task.tsnew file mode 100644index 00000000..0e7b929f--- /dev/null+++ b/apps/papra-server/src/modules/kv-store/tasks/purge-expired-kv-entries.task.ts@@ -0,0 +1,38 @@+import type { Config } from '../../config/config.types';+import type { TaskServices } from '../../tasks/tasks.services';+import type { KvStore } from '../kv-store.types';+import { createLogger } from '../../shared/logger/logger';+import { isNil } from '../../shared/utils';++const logger = createLogger({ namespace: 'kv-store:tasks:purgeExpiredEntries' });++export async function registerPurgeExpiredKvEntriesTask({ taskServices, kvStore, config }: { taskServices: TaskServices; kvStore: KvStore; config: Config }) {+ const taskName = 'purge-expired-kv-entries';+ const { cron, runOnStartup } = config.tasks.purgeExpiredKvEntries;++ const { purgeExpired } = kvStore;++ // Drivers that evict eagerly (e.g. in-memory) don't accumulate expired entries, so there's nothing to purge.+ if (isNil(purgeExpired)) {+ logger.debug('Kv-store driver does not require expired entry purging, skipping task registration');+ return;+ }++ taskServices.registerTask({+ taskName,+ handler: async () => {+ const { deletedCount } = await purgeExpired();++ logger.info({ deletedCount }, 'Purged expired kv-store entries');+ },+ });++ await taskServices.schedulePeriodicJob({+ scheduleId: `periodic-${taskName}`,+ taskName,+ cron,+ immediate: runOnStartup,+ });++ logger.info({ taskName, cron, runOnStartup }, 'Purge expired kv-store entries task registered');+}diff --git a/apps/papra-server/src/modules/tasks/tasks.config.ts b/apps/papra-server/src/modules/tasks/tasks.config.tsindex e3018acd..1c7fc715 100644--- a/apps/papra-server/src/modules/tasks/tasks.config.ts+++ b/apps/papra-server/src/modules/tasks/tasks.config.ts@@ -91,4 +91,18 @@ export const tasksConfig = {env: 'ORGANIZATIONS_PURGE_EXPIRED_ORGANIZATIONS_RUN_ON_STARTUP',},},+ purgeExpiredKvEntries: {+ cron: {+ doc: 'The cron schedule for the task to purge expired key-value store entries (only runs when the configured kv-store driver requires it, e.g. libsql)',+ schema: v.string(),+ default: '0 * * * *',+ env: 'KV_STORE_PURGE_EXPIRED_ENTRIES_CRON',+ },+ runOnStartup: {+ doc: 'Whether the task to purge expired key-value store entries should run on startup',+ schema: booleanishSchema,+ default: true,+ env: 'KV_STORE_PURGE_EXPIRED_ENTRIES_RUN_ON_STARTUP',+ },+ },} as const satisfies ConfigDefinition;diff --git a/apps/papra-server/src/modules/tasks/tasks.definitions.ts b/apps/papra-server/src/modules/tasks/tasks.definitions.tsindex 4f151b2d..2a461c23 100644--- a/apps/papra-server/src/modules/tasks/tasks.definitions.ts+++ b/apps/papra-server/src/modules/tasks/tasks.definitions.ts@@ -1,6 +1,7 @@import type { GlobalDependencies } from '../app/server.types';import { registerExtractDocumentFileContentTask } from '../documents/tasks/extract-document-file-content.task';import { registerHardDeleteExpiredDocumentsTask } from '../documents/tasks/hard-delete-expired-documents.task';+import { registerPurgeExpiredKvEntriesTask } from '../kv-store/tasks/purge-expired-kv-entries.task';import { registerExpireInvitationsTask } from '../organizations/tasks/expire-invitations.task';import { registerPurgeExpiredOrganizationsTask } from '../organizations/tasks/purge-expired-organizations.task';import { registerApplyTaggingRuleToDocumentsTask } from '../tagging-rules/tasks/apply-tagging-rule-to-documents.task';@@ -11,4 +12,5 @@ export async function registerTaskDefinitions(deps: GlobalDependencies) {await registerPurgeExpiredOrganizationsTask(deps);await registerExtractDocumentFileContentTask(deps);await registerApplyTaggingRuleToDocumentsTask(deps);+ await registerPurgeExpiredKvEntriesTask(deps);}