Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

feat(kv-store): add bulk purge of expired entries (#1120)

Corentin Thomassetdfb22ba5f403
diff --git a/apps/papra-server/src/modules/kv-store/drivers/kv-store-drivers.types.ts b/apps/papra-server/src/modules/kv-store/drivers/kv-store-drivers.types.ts
index 0ac7a587..82dac2b7 100644
--- a/apps/papra-server/src/modules/kv-store/drivers/kv-store-drivers.types.ts
+++ b/apps/papra-server/src/modules/kv-store/drivers/kv-store-drivers.types.ts
@@ -7,6 +7,8 @@ export type KvStoreDriver = {
get: (key: string) => Promise<JsonSerializableValue | undefined>;
set: (key: string, value: JsonSerializableValue, options?: { ttlMs?: number }) => Promise<void>;
delete: (key: string) => Promise<void>;
+ // Optional bulk purge of expired entries, only implemented by lazy-delete drivers (e.g. libsql) that would otherwise accumulate expired rows.
+ deleteExpired?: () => Promise<{ deletedCount: number }>;
};
export type KvStoreDriverFactory = (args: { config: Config; db: Database }) => KvStoreDriver;
diff --git a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.repository.ts b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.repository.ts
index 710f549a..dd9f7223 100644
--- a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.repository.ts
+++ b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.repository.ts
@@ -13,6 +13,7 @@ export function createKvStoreRepository({ db }: { db: Database }) {
upsertEntry,
deleteEntry,
deleteExpiredEntry,
+ deleteAllExpiredEntries,
},
{ db },
);
@@ -46,3 +47,17 @@ async function deleteExpiredEntry({ key, db, now = new Date() }: { key: string;
),
);
}
+
+async function deleteAllExpiredEntries({ db, now = new Date() }: { db: Database; now?: Date }) {
+ const deleted = await db
+ .delete(kvStoreTable)
+ .where(
+ and(
+ isNotNull(kvStoreTable.expiresAt),
+ lte(kvStoreTable.expiresAt, now),
+ ),
+ )
+ .returning({ key: kvStoreTable.key });
+
+ return { deletedCount: deleted.length };
+}
diff --git a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.test.ts b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.test.ts
index 0a909df2..4182f328 100644
--- a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.test.ts
+++ b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.test.ts
@@ -1,16 +1,45 @@
-import { describe } from 'vitest';
+import { describe, expect, test, vi } from 'vitest';
import { kvStoreMigration } from '../../../../migrations/list/0020-kv-store.migration';
import { setupDatabase } from '../../../app/database/database';
import { runKvStoreDriverTestSuite } from '../kv-store-drivers.test-suite';
import { createLibsqlKvStoreDriver } from './libsql.kv-store-driver';
+async function createDriver() {
+ const { db } = setupDatabase({ url: ':memory:' });
+ await kvStoreMigration.up({ db });
+
+ return { driver: createLibsqlKvStoreDriver({ db }) };
+}
+
describe('libsql.kv-store-driver', () => {
describe('createLibsqlKvStoreDriver', () => {
- runKvStoreDriverTestSuite(async () => {
- const { db } = setupDatabase({ url: ':memory:' });
- await kvStoreMigration.up({ db });
+ runKvStoreDriverTestSuite(createDriver);
+
+ describe('deleteExpired', () => {
+ test('removes only entries whose ttl has elapsed and returns how many were deleted', async () => {
+ vi.useFakeTimers();
+
+ const { driver } = await createDriver();
+
+ await driver.set('expired-1', 'value', { ttlMs: 1_000 });
+ await driver.set('expired-2', 'value', { ttlMs: 1_000 });
+ await driver.set('still-valid', 'value', { ttlMs: 10_000 });
+ await driver.set('no-ttl', 'value');
+
+ vi.advanceTimersByTime(1_001);
+
+ expect(await driver.deleteExpired!()).to.eql({ deletedCount: 2 });
+
+ // A second run finds nothing left to purge.
+ expect(await driver.deleteExpired!()).to.eql({ deletedCount: 0 });
+
+ expect(await driver.get('expired-1')).to.eql(undefined);
+ expect(await driver.get('expired-2')).to.eql(undefined);
+ expect(await driver.get('still-valid')).to.eql('value');
+ expect(await driver.get('no-ttl')).to.eql('value');
- return { driver: createLibsqlKvStoreDriver({ db }) };
+ vi.useRealTimers();
+ });
});
});
});
diff --git a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.ts b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.ts
index 055c2ba4..0f75e47c 100644
--- a/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.ts
+++ b/apps/papra-server/src/modules/kv-store/drivers/libsql/libsql.kv-store-driver.ts
@@ -42,5 +42,9 @@ export function createLibsqlKvStoreDriver({ db }: { db: Database }): KvStoreDriv
await repository.deleteEntry({ key });
},
+ deleteExpired: async (now = new Date()) => {
+ return repository.deleteAllExpiredEntries({ now });
+ },
+
};
}
diff --git a/apps/papra-server/src/modules/kv-store/kv-store.ts b/apps/papra-server/src/modules/kv-store/kv-store.ts
index fed91f3c..24eb56bd 100644
--- a/apps/papra-server/src/modules/kv-store/kv-store.ts
+++ b/apps/papra-server/src/modules/kv-store/kv-store.ts
@@ -17,6 +17,8 @@ export function createKvStore({
logger?: Logger;
}): KvStore {
return {
+ ...(driver.deleteExpired ? { purgeExpired: driver.deleteExpired } : {}),
+
defineScope: ({ prefix, schema, defaultTtlMs }) => {
const buildKey = (key: string) => joinKeyParts([prefix, key]);
diff --git a/apps/papra-server/src/modules/kv-store/kv-store.types.ts b/apps/papra-server/src/modules/kv-store/kv-store.types.ts
index 67cbea11..162623ba 100644
--- a/apps/papra-server/src/modules/kv-store/kv-store.types.ts
+++ b/apps/papra-server/src/modules/kv-store/kv-store.types.ts
@@ -16,4 +16,6 @@ export type KvStore = {
schema: TSchema;
defaultTtlMs?: number;
}) => KvStoreScope<TSchema>;
+ // Only present for lazy-delete drivers (e.g. libsql) that accumulate expired entries; eager drivers (e.g. in-memory) omit it.
+ purgeExpired?: () => Promise<{ deletedCount: number }>;
};
diff --git a/apps/papra-server/src/modules/kv-store/tasks/purge-expired-kv-entries.task.ts b/apps/papra-server/src/modules/kv-store/tasks/purge-expired-kv-entries.task.ts
new file mode 100644
index 00000000..0e7b929f
--- /dev/null
+++ b/apps/papra-server/src/modules/kv-store/tasks/purge-expired-kv-entries.task.ts
@@ -0,0 +1,38 @@
+import type { Config } from '../../config/config.types';
+import type { TaskServices } from '../../tasks/tasks.services';
+import type { KvStore } from '../kv-store.types';
+import { createLogger } from '../../shared/logger/logger';
+import { isNil } from '../../shared/utils';
+
+const logger = createLogger({ namespace: 'kv-store:tasks:purgeExpiredEntries' });
+
+export async function registerPurgeExpiredKvEntriesTask({ taskServices, kvStore, config }: { taskServices: TaskServices; kvStore: KvStore; config: Config }) {
+ const taskName = 'purge-expired-kv-entries';
+ const { cron, runOnStartup } = config.tasks.purgeExpiredKvEntries;
+
+ const { purgeExpired } = kvStore;
+
+ // Drivers that evict eagerly (e.g. in-memory) don't accumulate expired entries, so there's nothing to purge.
+ if (isNil(purgeExpired)) {
+ logger.debug('Kv-store driver does not require expired entry purging, skipping task registration');
+ return;
+ }
+
+ taskServices.registerTask({
+ taskName,
+ handler: async () => {
+ const { deletedCount } = await purgeExpired();
+
+ logger.info({ deletedCount }, 'Purged expired kv-store entries');
+ },
+ });
+
+ await taskServices.schedulePeriodicJob({
+ scheduleId: `periodic-${taskName}`,
+ taskName,
+ cron,
+ immediate: runOnStartup,
+ });
+
+ logger.info({ taskName, cron, runOnStartup }, 'Purge expired kv-store entries task registered');
+}
diff --git a/apps/papra-server/src/modules/tasks/tasks.config.ts b/apps/papra-server/src/modules/tasks/tasks.config.ts
index e3018acd..1c7fc715 100644
--- a/apps/papra-server/src/modules/tasks/tasks.config.ts
+++ b/apps/papra-server/src/modules/tasks/tasks.config.ts
@@ -91,4 +91,18 @@ export const tasksConfig = {
env: 'ORGANIZATIONS_PURGE_EXPIRED_ORGANIZATIONS_RUN_ON_STARTUP',
},
},
+ purgeExpiredKvEntries: {
+ cron: {
+ doc: 'The cron schedule for the task to purge expired key-value store entries (only runs when the configured kv-store driver requires it, e.g. libsql)',
+ schema: v.string(),
+ default: '0 * * * *',
+ env: 'KV_STORE_PURGE_EXPIRED_ENTRIES_CRON',
+ },
+ runOnStartup: {
+ doc: 'Whether the task to purge expired key-value store entries should run on startup',
+ schema: booleanishSchema,
+ default: true,
+ env: 'KV_STORE_PURGE_EXPIRED_ENTRIES_RUN_ON_STARTUP',
+ },
+ },
} as const satisfies ConfigDefinition;
diff --git a/apps/papra-server/src/modules/tasks/tasks.definitions.ts b/apps/papra-server/src/modules/tasks/tasks.definitions.ts
index 4f151b2d..2a461c23 100644
--- a/apps/papra-server/src/modules/tasks/tasks.definitions.ts
+++ b/apps/papra-server/src/modules/tasks/tasks.definitions.ts
@@ -1,6 +1,7 @@
import type { GlobalDependencies } from '../app/server.types';
import { registerExtractDocumentFileContentTask } from '../documents/tasks/extract-document-file-content.task';
import { registerHardDeleteExpiredDocumentsTask } from '../documents/tasks/hard-delete-expired-documents.task';
+import { registerPurgeExpiredKvEntriesTask } from '../kv-store/tasks/purge-expired-kv-entries.task';
import { registerExpireInvitationsTask } from '../organizations/tasks/expire-invitations.task';
import { registerPurgeExpiredOrganizationsTask } from '../organizations/tasks/purge-expired-organizations.task';
import { registerApplyTaggingRuleToDocumentsTask } from '../tagging-rules/tasks/apply-tagging-rule-to-documents.task';
@@ -11,4 +12,5 @@ export async function registerTaskDefinitions(deps: GlobalDependencies) {
await registerPurgeExpiredOrganizationsTask(deps);
await registerExtractDocumentFileContentTask(deps);
await registerApplyTaggingRuleToDocumentsTask(deps);
+ await registerPurgeExpiredKvEntriesTask(deps);
}