The minimalistic document archiving platform.
feat(storage): support document dates in storage keys (#1528)
Corentin Thomasset13a6d4a7b0d6
diff --git a/.changeset/itchy-ducks-shave.md b/.changeset/itchy-ducks-shave.mdnew file mode 100644index 00000000..ae38dbee--- /dev/null+++ b/.changeset/itchy-ducks-shave.md@@ -0,0 +1,5 @@+---+'@papra/app': patch+---++Added `document.date` and `document.createdAt` to storage key patterns, so you can organize files by document date or creation date. Documents without a date use `no-date` (and can be configured with the `default` transformer).diff --git a/apps/docs/src/content/docs/03-guides/09-storage-key-patterns.mdx b/apps/docs/src/content/docs/03-guides/09-storage-key-patterns.mdxindex cb7e22ce..fc77aa6e 100644--- a/apps/docs/src/content/docs/03-guides/09-storage-key-patterns.mdx+++ b/apps/docs/src/content/docs/03-guides/09-storage-key-patterns.mdx@@ -109,6 +109,8 @@ Multiple transformers can be chained, and each transformer receives the output o|-----------|-------------|----------------|| `document.id` | The unique document identifier | `doc_123456789012345678901234` || `document.name` | The document name (sanitized for safe filesystem use) | `invoice-2025.pdf` |+| `document.date` | The document date in ISO 8601 format, or `no-date` when unset | `2025-06-15T00:00:00.000Z` |+| `document.createdAt` | The document's creation timestamp in ISO 8601 format | `2025-06-15T14:30:00.000Z` || `organization.id` | The organization identifier | `org_123456789012345678901234` || `currentDate` | The current date and time in ISO 8601 format | `2025-06-15T14:30:00.000Z` || `currentDate.yyyy` | Current year (4 digits) | `2025` |@@ -133,6 +135,16 @@ The `document.name` expression automatically sanitizes the file name by replacin## Available Transformers+Both `document.date` and `document.createdAt` support the `formatDate` transformer:++```+{{organization.id}}/{{document.date | formatDate "{yyyy}/{MM}"}}/{{document.name}}+```++If the document date is unset, the expression resolves to `no-date`, even with date formatting. Use `default` after formatting to customize the fallback, for example `{{document.date | formatDate | default "undated"}}`.++The `document.createdAt` value stays fixed on subsequent storage-key synchronizations. When storage-key synchronization is enabled, setting, changing, or clearing the document date also triggers synchronization.+Transformers modify the value of an expression. They are applied using the pipe (`|`) operator after the expression name.| Transformer | Arguments | Description | Example |diff --git a/apps/papra-server/src/modules/documents/document-storage.usecases.test.ts b/apps/papra-server/src/modules/documents/document-storage.usecases.test.tsindex d05a98f4..0f36af7d 100644--- a/apps/papra-server/src/modules/documents/document-storage.usecases.test.ts+++ b/apps/papra-server/src/modules/documents/document-storage.usecases.test.ts@@ -24,6 +24,8 @@ const baseStoragePatternConfig = {const documentContext = {documentId: 'doc_1',documentName: 'invoice.pdf',+ documentDate: null,+ documentCreatedAt: new Date('2025-12-01T00:00:00.000Z'),organizationId: 'org_1',now: new Date('2026-01-01T00:00:00.000Z'),};@@ -76,6 +78,27 @@ async function setupSync({ sourceStorageKey = 'old.pdf', isEncryptionEnabled = f}describe('syncDocumentStorageKey', () => {+ test('the creation date in a storage key stays fixed when the current date changes', async () => {+ const { args, dependencies, document, getDocument, keys } = await setupSync();+ const { clock } = createTestClock({ now: '2030-01-01T00:00:00Z' });+ const sync = buildSyncDocumentStorageKey({+ ...dependencies,+ clock,+ storagePatternConfig: {+ ...baseStoragePatternConfig,+ storageKeyPattern: '{{document.createdAt}}/{{document.name}}',+ },+ });++ await sync(args);+ clock.setNow('2031-01-01T00:00:00Z');+ await sync(args);++ const storageKey = `${document.createdAt.toISOString()}/invoice.pdf`;+ expect((await getDocument())?.originalStorageKey).toEqual(storageKey);+ expect(keys()).toEqual([storageKey]);+ });+test('re-evaluates current dates and random expressions on subsequent updates', async () => {const { args, dependencies, keys, getDocument } = await setupSync();const { clock } = createTestClock({ now: '2026-01-01T00:00:00Z' });@@ -413,6 +436,21 @@ describe('syncDocumentStorageKey', () => {describe('document-storage usecases', () => {describe('createDocumentStorageKey', () => {+ test('uses the supplied document dates when generating a key for an existing document', async () => {+ await expect(+ createDocumentStorageKey({+ ...documentContext,+ documentDate: new Date('2024-03-12T00:00:00.000Z'),+ storagePatternConfig: {+ ...baseStoragePatternConfig,+ storageKeyPattern:+ '{{document.date | formatDate}}/{{document.createdAt | formatDate}}/{{document.name}}',+ },+ documentsStorageService: { fileExists: async () => false },+ }),+ ).resolves.toEqual({ storageKey: '2024-03-12/2025-12-01/invoice.pdf' });+ });+test('uses the legacy document key without checking for collisions', async () => {const fileExists = vi.fn(async () => false);diff --git a/apps/papra-server/src/modules/documents/document-storage.usecases.ts b/apps/papra-server/src/modules/documents/document-storage.usecases.tsindex 1b2cf0aa..08110a68 100644--- a/apps/papra-server/src/modules/documents/document-storage.usecases.ts+++ b/apps/papra-server/src/modules/documents/document-storage.usecases.ts@@ -55,6 +55,8 @@ export function buildSyncDocumentStorageKey({documentId,organizationId,documentName: document.name,+ documentDate: document.documentDate,+ documentCreatedAt: document.createdAt,now: new Date(clock.now().epochMilliseconds),});@@ -107,6 +109,8 @@ export async function createDocumentStorageKey({storagePatternConfig,documentId,documentName,+ documentDate,+ documentCreatedAt,organizationId,documentsStorageService,logger,@@ -115,6 +119,8 @@ export async function createDocumentStorageKey({storagePatternConfig: StoragePatternConfig;documentId: string;documentName: string;+ documentDate: Date | null;+ documentCreatedAt: Date;organizationId: string;documentsStorageService: Pick<StorageService, 'fileExists'>;logger?: Logger;@@ -141,6 +147,8 @@ export async function createDocumentStorageKey({storageKeyPattern,documentId,documentName,+ documentDate,+ documentCreatedAt,organizationId,now,});diff --git a/apps/papra-server/src/modules/documents/documents.events.types.ts b/apps/papra-server/src/modules/documents/documents.events.types.tsindex 58e35839..85bc3e26 100644--- a/apps/papra-server/src/modules/documents/documents.events.types.ts+++ b/apps/papra-server/src/modules/documents/documents.events.types.ts@@ -11,6 +11,7 @@ export type DocumentEvents = {name?: string;content?: string;notes?: string;+ documentDate?: Date | null;};};'document.deleted': { documentId: string; organizationId: string }; // Hard deleted from trashdiff --git a/apps/papra-server/src/modules/documents/documents.usecases.test.ts b/apps/papra-server/src/modules/documents/documents.usecases.test.tsindex 46b9bbd1..16a212ce 100644--- a/apps/papra-server/src/modules/documents/documents.usecases.test.ts+++ b/apps/papra-server/src/modules/documents/documents.usecases.test.ts@@ -35,6 +35,41 @@ import { createInMemoryStorageService } from '../storage/storage.test-utils';describe('documents usecases', () => {describe('createDocument', () => {+ test('a new document key uses its persisted creation timestamp and the missing-date fallback', async () => {+ const { db } = await createInMemoryDatabase({+ organizations: [{ id: 'organization-1', name: 'Organization 1' }],+ });+ const documentsStorageService = createInMemoryStorageService();+ const createDocument = createDocumentCreationUsecase({+ db,+ config: overrideConfig({+ organizationPlans: { isFreePlanUnlimited: true },+ documentsStorage: {+ pattern: {+ useLegacyStorageKeyDefinitionSystem: false,+ storageKeyPattern:+ '{{document.createdAt}}/{{document.date | formatDate}}/{{document.name}}',+ },+ },+ }),+ documentsStorageService,+ taskServices: createInMemoryTaskServices(),+ eventServices: createTestEventServices(),+ });++ const { document } = await createDocument({+ fileStream: createReadableStream({ content: 'Hello, world!' }),+ fileName: 'file.txt',+ mimeType: 'text/plain',+ organizationId: 'organization-1',+ });++ const storageKey = `${document.createdAt.toISOString()}/no-date/file.txt`;+ expect(document.originalStorageKey).toEqual(storageKey);+ expect(await db.select().from(documentsTable)).toEqual([document]);+ expect(await documentsStorageService.fileExists({ storageKey })).toEqual(true);+ });+test('creating a document save the file to the storage and registers a record in the db', async () => {const taskServices = createInMemoryTaskServices();const { db } = await createInMemoryDatabase({diff --git a/apps/papra-server/src/modules/documents/documents.usecases.ts b/apps/papra-server/src/modules/documents/documents.usecases.tsindex 3e57d258..b68072db 100644--- a/apps/papra-server/src/modules/documents/documents.usecases.ts+++ b/apps/papra-server/src/modules/documents/documents.usecases.ts@@ -108,9 +108,12 @@ export async function createDocument({});const documentId = generateDocumentId();+ const createdAt = new Date();const { storageKey } = await createDocumentStorageKey({documentId,documentName: fileName,+ documentDate: null,+ documentCreatedAt: createdAt,organizationId,documentsStorageService,storagePatternConfig,@@ -172,6 +175,7 @@ export async function createDocument({logger,}): await createNewDocument({+ createdAt,newFileStorageContext: { storageKey, ...encryptionMetadata },fileName,size,@@ -309,6 +313,7 @@ async function handleExistingDocument({}async function createNewDocument({+ createdAt,fileName,size,mimeType,@@ -328,6 +333,7 @@ async function createNewDocument({isContentExtractionEnabled = true,logger,}: {+ createdAt: Date;fileName: string;size: number;mimeType: string;@@ -372,6 +378,7 @@ async function createNewDocument({const [result, error] = await safely(documentsRepository.saveOrganizationDocument({id: documentId,+ createdAt,name: fileName,organizationId,originalName: fileName,diff --git a/apps/papra-server/src/modules/documents/events/storage-key.document-updated.test.ts b/apps/papra-server/src/modules/documents/events/storage-key.document-updated.test.tsnew file mode 100644index 00000000..3deac3a4--- /dev/null+++ b/apps/papra-server/src/modules/documents/events/storage-key.document-updated.test.ts@@ -0,0 +1,86 @@+import { createNoopLogger } from '@crowlog/logger';+import { describe, expect, test } from 'vitest';+import { createInMemoryDatabase } from '../../app/database/database.test-utils';+import { createEventServices } from '../../app/events/events.services';+import { overrideConfig } from '../../config/config.test-utils';+import { nextTick } from '../../shared/async/defer.test-utils';+import { createReadableStream } from '../../shared/streams/readable-stream';+import { createInMemoryStorageService } from '../../storage/storage.test-utils';+import { createDocumentsRepository } from '../documents.repository';+import { updateDocument } from '../documents.usecases';+import { registerSyncDocumentStorageKeyHandler } from './storage-key.document-updated';++async function setup() {+ const { db } = await createInMemoryDatabase({+ organizations: [{ id: 'org_1', name: 'Organization' }],+ });+ const documentsRepository = createDocumentsRepository({ db });+ const documentsStorageService = createInMemoryStorageService();+ const eventServices = createEventServices({ logger: createNoopLogger() });+ const { document } = await documentsRepository.saveOrganizationDocument({+ id: 'doc_1',+ organizationId: 'org_1',+ name: 'invoice.pdf',+ originalName: 'invoice.pdf',+ originalStorageKey: 'old.pdf',+ originalSha256Hash: 'hash',+ mimeType: 'application/pdf',+ });+ await documentsStorageService.saveFile({+ storageKey: document.originalStorageKey,+ fileName: document.name,+ mimeType: document.mimeType,+ fileStream: createReadableStream({ content: 'document bytes' }),+ });+ registerSyncDocumentStorageKeyHandler({+ db,+ documentsStorageService,+ eventServices,+ config: overrideConfig({+ documentsStorage: {+ pattern: {+ isStorageKeySyncEnabled: true,+ useLegacyStorageKeyDefinitionSystem: false,+ storageKeyPattern: '{{document.date | formatDate}}/{{document.name}}',+ },+ },+ }),+ });++ const args = { documentId: document.id, organizationId: document.organizationId };+ return {+ update: async (changes: Parameters<typeof updateDocument>[0]['changes']) =>+ updateDocument({ ...args, changes, documentsRepository, eventServices }),+ getStorageKey: async () =>+ (await documentsRepository.getDocumentById(args)).document?.originalStorageKey,+ keys: () => [...documentsStorageService._getStorage().keys()],+ };+}++describe('storage-key document.updated handler', () => {+ test('setting, changing, and clearing a document date move the file to the matching key', async () => {+ const { update, getStorageKey, keys } = await setup();++ await update({ documentDate: new Date('2024-03-12T00:00:00.000Z') });+ await expect.poll(getStorageKey).toEqual('2024-03-12/invoice.pdf');+ await expect.poll(keys).toEqual(['2024-03-12/invoice.pdf']);++ await update({ documentDate: new Date('2025-06-15T00:00:00.000Z') });+ await expect.poll(getStorageKey).toEqual('2025-06-15/invoice.pdf');+ await expect.poll(keys).toEqual(['2025-06-15/invoice.pdf']);++ await update({ documentDate: null });+ await expect.poll(getStorageKey).toEqual('no-date/invoice.pdf');+ await expect.poll(keys).toEqual(['no-date/invoice.pdf']);+ });++ test('content and notes updates do not synchronize the storage key', async () => {+ const { update, getStorageKey, keys } = await setup();++ await update({ content: 'updated content', notes: 'updated notes' });+ await nextTick();++ expect(await getStorageKey()).toEqual('old.pdf');+ expect(keys()).toEqual(['old.pdf']);+ });+});diff --git a/apps/papra-server/src/modules/documents/events/storage-key.document-updated.ts b/apps/papra-server/src/modules/documents/events/storage-key.document-updated.tsindex 9070ae70..7a2799c1 100644--- a/apps/papra-server/src/modules/documents/events/storage-key.document-updated.ts+++ b/apps/papra-server/src/modules/documents/events/storage-key.document-updated.ts@@ -26,7 +26,7 @@ export function registerSyncDocumentStorageKeyHandler({eventName: 'document.updated',handlerName: 'sync-document-storage-key',handler: async ({ document, changes }) => {- if (changes.name === undefined) {+ if (changes.name === undefined && changes.documentDate === undefined) {return;}diff --git a/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.definitions.ts b/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.definitions.tsindex 877bd5e2..92b8a47e 100644--- a/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.definitions.ts+++ b/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.definitions.ts@@ -11,6 +11,11 @@ import { ensureSafeFileName } from '../documents.models';export const expressionsDefinitions: Record<string, StoragePatternExpressionDefinition> = {'document.id': { resolve: (context) => context.documentId },'document.name': { resolve: (context) => ensureSafeFileName(context.documentName) },+ 'document.date': {+ resolve: (context) => context.documentDate?.toISOString(),+ fallback: 'no-date',+ },+ 'document.createdAt': { resolve: (context) => context.documentCreatedAt.toISOString() },'organization.id': { resolve: (context) => context.organizationId },'currentDate': { resolve: (context) => context.now.toISOString() },'random': { resolve: () => generateRandomString({ length: 8 }) },diff --git a/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.models.test.ts b/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.models.test.tsindex 859d3df2..8285735b 100644--- a/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.models.test.ts+++ b/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.models.test.ts@@ -10,6 +10,8 @@ describe('storage-pattern models', () => {const context: StoragePatternInterpolationContext = {documentId: 'doc_012345678901234567890123',documentName: 'My Document.pdf',+ documentDate: null,+ documentCreatedAt: new Date('2025-05-15T12:34:56.789Z'),organizationId: 'org_012345678901234567890123',now: new Date('2025-05-15T12:34:56.789Z'),};diff --git a/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.types.ts b/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.types.tsindex 5b785632..ac33ef7d 100644--- a/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.types.ts+++ b/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.types.ts@@ -3,6 +3,8 @@ import type { Config } from '../../config/config.types';export type StoragePatternInterpolationContext = {documentId: string;documentName: string;+ documentDate: Date | null;+ documentCreatedAt: Date;organizationId: string;now: Date;};diff --git a/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.usecases.test.ts b/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.usecases.test.tsindex efbba634..40186c54 100644--- a/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.usecases.test.ts+++ b/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.usecases.test.ts@@ -7,6 +7,8 @@ describe('storage-pattern usecases', () => {const context: StoragePatternInterpolationContext = {documentId: 'doc_012345678901234567890123',documentName: 'My Document.pdf',+ documentDate: new Date('2024-03-12T00:00:00.000Z'),+ documentCreatedAt: new Date('2025-01-02T10:20:30.456Z'),organizationId: 'org_012345678901234567890123',now: new Date('2025-05-15T12:34:56.789Z'),};@@ -77,6 +79,45 @@ describe('storage-pattern usecases', () => {}});+ test('document dates resolve to their ISO timestamps independently of the current date', () => {+ expect(+ buildStorageKey({+ ...context,+ storageKeyPattern: '{{document.date}}/{{document.createdAt}}',+ }),+ ).toEqual({ storageKey: '2024-03-12T00:00:00.000Z/2025-01-02T10:20:30.456Z' });+ });++ test('document dates support date formatting', () => {+ expect(+ buildStorageKey({+ ...context,+ storageKeyPattern:+ '{{document.date | formatDate "{yyyy}/{MM}"}}/{{document.createdAt | formatDate}}',+ }),+ ).toEqual({ storageKey: '2024/03/2025-01-02' });+ });++ test('an unset document date falls back to no-date with or without date formatting', () => {+ expect(+ buildStorageKey({+ ...context,+ documentDate: null,+ storageKeyPattern: '{{document.date}}/{{document.date | formatDate "{yyyy}/{MM}"}}',+ }),+ ).toEqual({ storageKey: 'no-date/no-date' });+ });++ test('a custom default overrides the missing document date fallback', () => {+ expect(+ buildStorageKey({+ ...context,+ documentDate: null,+ storageKeyPattern: '{{document.date | formatDate | default "undated" | uppercase}}',+ }),+ ).toEqual({ storageKey: 'UNDATED' });+ });+test('unrecognized expressions throw an error', () => {expect(() =>buildStorageKey({@@ -116,6 +157,15 @@ describe('storage-pattern usecases', () => {});describe('isStoragePatternValid', () => {+ test('document date and creation date expressions with formatting are valid', () => {+ expect(+ isStoragePatternValid({+ storageKeyPattern:+ '{{document.date | formatDate "{yyyy}"}}/{{document.createdAt | formatDate}}/{{document.name}}',+ }),+ ).toEqual({ isValid: true });+ });+test('a pattern with a default argument is valid', () => {expect(isStoragePatternValid({diff --git a/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.usecases.ts b/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.usecases.tsindex c2623119..f3894098 100644--- a/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.usecases.ts+++ b/apps/papra-server/src/modules/documents/storage-patterns/storage-pattern.usecases.ts@@ -58,6 +58,8 @@ export function isStoragePatternValid({storageKeyPattern,documentId: DUMMY_DOCUMENT_ID,documentName: 'my-document.pdf',+ documentDate: new Date(),+ documentCreatedAt: new Date(),organizationId: DUMMY_ORGANIZATION_ID,now: new Date(),});diff --git a/apps/papra-server/src/scripts/migrate-document-storage.script.ts b/apps/papra-server/src/scripts/migrate-document-storage.script.tsindex 72449740..5f59fd6b 100644--- a/apps/papra-server/src/scripts/migrate-document-storage.script.ts+++ b/apps/papra-server/src/scripts/migrate-document-storage.script.ts@@ -81,6 +81,8 @@ export async function migrateDocumentStorage({originalStorageKey: documentsTable.originalStorageKey,organizationId: documentsTable.organizationId,originalName: documentsTable.originalName,+ documentDate: documentsTable.documentDate,+ createdAt: documentsTable.createdAt,mimeType: documentsTable.mimeType,fileEncryptionKeyWrapped: documentsTable.fileEncryptionKeyWrapped,fileEncryptionKekVersion: documentsTable.fileEncryptionKekVersion,@@ -131,6 +133,8 @@ export async function migrateDocumentStorage({documentId: id,organizationId,documentName: originalName,+ documentDate: document.documentDate,+ documentCreatedAt: document.createdAt,documentsStorageService: toStorageService,logger: createNoopLogger(),});