The minimalistic document archiving platform.
feat(webhooks): support batching multiple payloads per event (#1068)
Corentin Thomassetdc78b5ed7d11
diff --git a/apps/papra-server/src/modules/documents/events/webhooks.document-created.ts b/apps/papra-server/src/modules/documents/events/webhooks.document-created.tsindex 26673c4f..f30713b6 100644--- a/apps/papra-server/src/modules/documents/events/webhooks.document-created.ts+++ b/apps/papra-server/src/modules/documents/events/webhooks.document-created.ts@@ -15,13 +15,13 @@ export function registerTriggerWebhooksOnDocumentCreatedHandler({await webhookTriggerServices.triggerWebhooks({organizationId: document.organizationId,event: 'document:created',- payload: {+ payloads: [{documentId: document.id,organizationId: document.organizationId,name: document.name,createdAt: document.createdAt,updatedAt: document.updatedAt,- },+ }],});},});diff --git a/apps/papra-server/src/modules/documents/events/webhooks.document-updated.ts b/apps/papra-server/src/modules/documents/events/webhooks.document-updated.tsindex ae0bff0f..c339ec69 100644--- a/apps/papra-server/src/modules/documents/events/webhooks.document-updated.ts+++ b/apps/papra-server/src/modules/documents/events/webhooks.document-updated.ts@@ -15,11 +15,11 @@ export function registerTriggerWebhooksOnDocumentUpdatedHandler({await webhookTriggerServices.triggerWebhooks({organizationId: document.organizationId,event: 'document:updated',- payload: {+ payloads: [{documentId: document.id,organizationId: document.organizationId,...changes,- },+ }],});},});diff --git a/apps/papra-server/src/modules/documents/events/webhooks.documents-trashed.ts b/apps/papra-server/src/modules/documents/events/webhooks.documents-trashed.tsindex 06451ddb..38caefc4 100644--- a/apps/papra-server/src/modules/documents/events/webhooks.documents-trashed.ts+++ b/apps/papra-server/src/modules/documents/events/webhooks.documents-trashed.ts@@ -1,6 +1,5 @@import type { EventServices } from '../../app/events/events.services';import type { WebhookTriggerServices } from '../../webhooks/webhooks.trigger.services';-import pLimit from 'p-limit';export function registerTriggerWebhooksOnDocumentsTrashedHandler({eventServices,@@ -13,22 +12,14 @@ export function registerTriggerWebhooksOnDocumentsTrashedHandler({eventName: 'documents.trashed',handlerName: 'trigger-webhooks',async handler({ documentIds, organizationId }) {- const limit = pLimit(10);-- await Promise.all(- documentIds.map(async documentId =>- limit(async () =>- webhookTriggerServices.triggerWebhooks({- organizationId,- event: 'document:deleted',- payload: {- documentId,- organizationId,- },- }),- ),- ),- );+ await webhookTriggerServices.triggerWebhooks({+ organizationId,+ event: 'document:deleted',+ payloads: documentIds.map(documentId => ({+ documentId,+ organizationId,+ })),+ });},});}diff --git a/apps/papra-server/src/modules/shared/ssrf/ssrf.services.test.ts b/apps/papra-server/src/modules/shared/ssrf/ssrf.services.test.tsindex 7d4affe3..e06899d1 100644--- a/apps/papra-server/src/modules/shared/ssrf/ssrf.services.test.ts+++ b/apps/papra-server/src/modules/shared/ssrf/ssrf.services.test.ts@@ -1,6 +1,6 @@import { describe, expect, test } from 'vitest';import { createTestLogger } from '../logger/logger.test-utils';-import { isUrlSsrfSafe } from './ssrf.services';+import { createCachedIsUrlSsrfSafeFunction, isUrlSsrfSafe } from './ssrf.services';describe('ssrf services', () => {describe('isUrlSsrfSafe', () => {@@ -103,8 +103,7 @@ describe('ssrf services', () => {const logs = getLogs({ excludeTimestampMs: true });expect(logs.length).toBe(1);- expect(logs[0]).to.contain({ level: 'error', message: 'Error while checking if URL is SSRF safe' });- expect(logs[0]?.data.error).to.contain({ message: 'Invalid URL', name: 'TypeError' });+ expect(logs[0]).to.contain({ level: 'error', message: 'Invalid URL provided, cannot extract hostname' });});test('given an url with an hostname, if the DNS lookup throws an error, the function should return false and not throw', async () => {@@ -118,9 +117,32 @@ describe('ssrf services', () => {const logs = getLogs({ excludeTimestampMs: true });expect(logs.length).toBe(1);- expect(logs[0]).to.contain({ level: 'error', message: 'Error while checking if URL is SSRF safe' });+ expect(logs[0]).to.contain({ level: 'error', message: 'Error while checking if hostname is SSRF safe' });expect(logs[0]?.data.error).to.contain({ message: 'DNS lookup failed', name: 'Error' });});});});++ describe('createCachedIsUrlSsrfSafeFunction', () => {+ describe(`when checking if multiple urls are SSRF safe, to avoid performing multiple DNS lookups for the same hostnames, a caching layer can be used+ note that this should be used with caution as it can be vulnerable to TOCTOU DNS rebinding attacks in the window between the cache check and the actual request+ so to be used for one-time checks of webhook urls for example, but not for every request to a given url`, () => {+ test('when an hostname is checked multiple times, the DNS lookup should only be performed once', async () => {+ let dnsLookupCallCount = 0;++ const dnsLookup = async () => {+ dnsLookupCallCount++;+ return { addresses: [{ address: '8.8.8.8', family: 4 }] };+ };++ const cachedIsUrlSsrfSafe = createCachedIsUrlSsrfSafeFunction({ dnsLookup });++ await cachedIsUrlSsrfSafe({ url: 'http://example.com/foo/bar' });+ await cachedIsUrlSsrfSafe({ url: 'http://example.com/foo/bar' });+ await cachedIsUrlSsrfSafe({ url: 'http://example.com/other/path' });++ expect(dnsLookupCallCount).toBe(1);+ });+ });+ });});diff --git a/apps/papra-server/src/modules/shared/ssrf/ssrf.services.ts b/apps/papra-server/src/modules/shared/ssrf/ssrf.services.tsindex 11731c10..a522f871 100644--- a/apps/papra-server/src/modules/shared/ssrf/ssrf.services.ts+++ b/apps/papra-server/src/modules/shared/ssrf/ssrf.services.ts@@ -2,6 +2,8 @@ import type { Logger } from '@crowlog/logger';import { isIP } from 'node:net';import { dnsLookupHostname } from '../dns/dns.services';import { createLogger } from '../logger/logger';+import { getUrlHostname } from '../urls/urls.models';+import { isNil } from '../utils';import { isIpSsrfSafe } from './ssrf.models';export async function isUrlSsrfSafe({@@ -15,9 +17,28 @@ export async function isUrlSsrfSafe({dnsLookup?: (args: { hostname: string }) => Promise<{ addresses: { address: string; family: number }[] }>;logger?: Logger;}): Promise<boolean> {- try {- const { hostname } = new URL(url);+ const hostname = getUrlHostname(url);++ if (isNil(hostname)) {+ logger.error({ url }, 'Invalid URL provided, cannot extract hostname');+ return false;+ }+ return isHostnameSsrfSafe({ hostname, allowedHostnames, dnsLookup, logger });+}++async function isHostnameSsrfSafe({+ hostname,+ allowedHostnames,+ dnsLookup = dnsLookupHostname,+ logger = createLogger({ namespace: 'ssrf.services' }),+}: {+ hostname: string;+ allowedHostnames?: Set<string>;+ dnsLookup?: (args: { hostname: string }) => Promise<{ addresses: { address: string; family: number }[] }>;+ logger?: Logger;+}): Promise<boolean> {+ try {if (allowedHostnames && allowedHostnames.has(hostname)) {return true;}@@ -30,7 +51,40 @@ export async function isUrlSsrfSafe({return addresses.length > 0 && addresses.every(({ address }) => isIpSsrfSafe(address));} catch (error) {- logger.error({ error, url }, 'Error while checking if URL is SSRF safe');+ logger.error({ error, hostname }, 'Error while checking if hostname is SSRF safe');return false;}}++export function createCachedIsUrlSsrfSafeFunction({+ allowedHostnames,+ dnsLookup,+ logger = createLogger({ namespace: 'ssrf.services' }),+}: {+ allowedHostnames?: Set<string>;+ dnsLookup?: (args: { hostname: string }) => Promise<{ addresses: { address: string; family: number }[] }>;+ logger?: Logger;+}) {+ const hostnameCache = new Map<string, boolean>();++ return async ({ url }: { url: string }) => {+ const hostname = getUrlHostname(url);++ if (isNil(hostname)) {+ logger.error({ url }, 'Invalid URL provided, cannot extract hostname');+ return false;+ }++ const cachedResult = hostnameCache.get(hostname);++ if (cachedResult !== undefined) {+ return cachedResult;+ }++ const isSafe = await isHostnameSsrfSafe({ hostname, allowedHostnames, dnsLookup, logger });++ hostnameCache.set(hostname, isSafe);++ return isSafe;+ };+}diff --git a/apps/papra-server/src/modules/shared/urls/urls.models.test.ts b/apps/papra-server/src/modules/shared/urls/urls.models.test.tsnew file mode 100644index 00000000..4c5c6ea1--- /dev/null+++ b/apps/papra-server/src/modules/shared/urls/urls.models.test.ts@@ -0,0 +1,23 @@+import { describe, expect, test } from 'vitest';+import { getUrlHostname } from './urls.models';++describe('urls.models', () => {+ describe('getUrlHostname', () => {+ test('safely get the hostname of a url', () => {+ expect(getUrlHostname('http://example.com/foo/bar')).to.eql('example.com');+ expect(getUrlHostname('https://example.com/foo/bar')).to.eql('example.com');+ expect(getUrlHostname('ftp://example.com/foo/bar')).to.eql('example.com');+ expect(getUrlHostname('http://example.com:8080/foo/bar')).to.eql('example.com');+ expect(getUrlHostname('http://user:[email protected]/foo/bar')).to.eql('example.com');+ expect(getUrlHostname('http://127.0.0.1/foo/bar')).to.eql('127.0.0.1');+ expect(getUrlHostname('http://[2001:db8::1]/foo/bar')).to.eql('[2001:db8::1]');+ });++ test('given an invalid url, return null', () => {+ expect(getUrlHostname('not a url')).to.eql(null);+ expect(getUrlHostname('http://')).to.eql(null);+ expect(getUrlHostname('foo/bar')).to.eql(null);+ expect(getUrlHostname('')).to.eql(null);+ });+ });+});diff --git a/apps/papra-server/src/modules/shared/urls/urls.models.ts b/apps/papra-server/src/modules/shared/urls/urls.models.tsnew file mode 100644index 00000000..b3c03316--- /dev/null+++ b/apps/papra-server/src/modules/shared/urls/urls.models.ts@@ -0,0 +1,7 @@+export function getUrlHostname(url: string) {+ try {+ return new URL(url).hostname;+ } catch {+ return null;+ }+}diff --git a/apps/papra-server/src/modules/tags/tags.routes.ts b/apps/papra-server/src/modules/tags/tags.routes.tsindex 0d255f82..c12b31e4 100644--- a/apps/papra-server/src/modules/tags/tags.routes.ts+++ b/apps/papra-server/src/modules/tags/tags.routes.ts@@ -234,7 +234,7 @@ function setupRemoveTagFromDocumentRoute({ app, db, webhookTriggerServices }: RowebhookTriggerServices.deferTriggerWebhooks({organizationId,event: 'document:tag:removed',- payload: { documentId, organizationId, tagId, tagName: tag.name },+ payloads: [{ documentId, organizationId, tagId, tagName: tag.name }],});deferRegisterDocumentActivityLog({diff --git a/apps/papra-server/src/modules/tags/tags.usecases.ts b/apps/papra-server/src/modules/tags/tags.usecases.tsindex 6324f192..d2cf0a8a 100644--- a/apps/papra-server/src/modules/tags/tags.usecases.ts+++ b/apps/papra-server/src/modules/tags/tags.usecases.ts@@ -70,7 +70,7 @@ export async function addTagToDocument({webhookTriggerServices.deferTriggerWebhooks({organizationId,event: 'document:tag:added',- payload: { documentId, organizationId, tagId, tagName: tag.name },+ payloads: [{ documentId, organizationId, tagId, tagName: tag.name }],});deferRegisterDocumentActivityLog({diff --git a/apps/papra-server/src/modules/webhooks/webhooks.types.ts b/apps/papra-server/src/modules/webhooks/webhooks.types.tsindex 4007ccc1..ef7de5b5 100644--- a/apps/papra-server/src/modules/webhooks/webhooks.types.ts+++ b/apps/papra-server/src/modules/webhooks/webhooks.types.ts@@ -1,3 +1,4 @@+import type { WebhookPayloads } from '@papra/webhooks';import type { Config } from '../config/config.types';import type { webhookDeliveriesTable, webhookEventsTable, webhooksTable } from './webhooks.tables';@@ -7,3 +8,10 @@ export type WebhookDelivery = typeof webhookDeliveriesTable.$inferSelect;export type WebhookDeliveryInsert = typeof webhookDeliveriesTable.$inferInsert;export type WebhooksConfig = Config['webhooks'];++export type WebhookMultiplePayloads = {+ [E in WebhookPayloads['event']]: {+ event: E;+ payloads: Extract<WebhookPayloads, { event: E }>['payload'][];+ };+}[WebhookPayloads['event']];diff --git a/apps/papra-server/src/modules/webhooks/webhooks.usecases.test.ts b/apps/papra-server/src/modules/webhooks/webhooks.usecases.test.tsindex 66e3cc8a..04d8fc3d 100644--- a/apps/papra-server/src/modules/webhooks/webhooks.usecases.test.ts+++ b/apps/papra-server/src/modules/webhooks/webhooks.usecases.test.ts@@ -164,16 +164,18 @@ describe('webhook usecases', () => {});describe('triggerWebhooks', () => {- test('when SSRF protection is enabled, triggering a webhook with a private IP URL does not call the webhook service', async () => {+ test('when SSRF protection is enabled, webhooks with unsafe URLs are filtered out and others still receive the event', async () => {const { db } = await createInMemoryDatabase({organizations: [{ id: 'org_1', name: 'Organization 1' },],webhooks: [- { id: 'wbh_1', name: 'Test Webhook', url: 'https://192.168.1.1/webhook', organizationId: 'org_1', enabled: true },+ { id: 'wbh_1', name: 'Unsafe Webhook', url: 'https://192.168.1.1/webhook', organizationId: 'org_1', enabled: true },+ { id: 'wbh_2', name: 'Safe Webhook', url: 'https://example.com/webhook', organizationId: 'org_1', enabled: true },],webhookEvents: [{ id: 'wbh_ev_1', webhookId: 'wbh_1', eventName: 'document:created' },+ { id: 'wbh_ev_2', webhookId: 'wbh_2', eventName: 'document:created' },],});@@ -181,17 +183,17 @@ describe('webhook usecases', () => {const { logger } = createTestLogger();const triggerWebhookServiceArgs: unknown[] = [];- await expect(triggerWebhooks({+ await triggerWebhooks({webhookRepository,organizationId: 'org_1',event: 'document:created',- payload: {+ payloads: [{documentId: 'doc_1',organizationId: 'org_1',name: 'Document 1',createdAt: new Date('2025-01-01'),updatedAt: new Date('2025-01-01'),- },+ }],logger,triggerWebhookService: async (args) => {triggerWebhookServiceArgs.push(args);@@ -201,9 +203,10 @@ describe('webhook usecases', () => {isSsrfProtectionEnabled: true,webhookUrlAllowedHostnames: new Set<string>(),},- })).rejects.toThrow('The provided URL is not safe to perform requests to');+ });- expect(triggerWebhookServiceArgs).to.eql([]);+ expect(triggerWebhookServiceArgs).to.have.lengthOf(1);+ expect((triggerWebhookServiceArgs[0] as { webhookUrl: string }).webhookUrl).to.eql('https://example.com/webhook');});test('when an organization has webhooks enabled for an event, the configured urls are called with the event payload', async () => {@@ -249,7 +252,7 @@ describe('webhook usecases', () => {webhookRepository,organizationId: 'org_1',event: 'document:created',- payload: eventPayload,+ payloads: [eventPayload],logger,now: new Date('2025-05-04'),triggerWebhookService: async (args) => {diff --git a/apps/papra-server/src/modules/webhooks/webhooks.usecases.ts b/apps/papra-server/src/modules/webhooks/webhooks.usecases.tsindex fc719139..c170caf9 100644--- a/apps/papra-server/src/modules/webhooks/webhooks.usecases.ts+++ b/apps/papra-server/src/modules/webhooks/webhooks.usecases.ts@@ -1,12 +1,12 @@import type { EventName, WebhookPayloads } from '@papra/webhooks';import type { Logger } from '../shared/logger/logger';import type { WebhookRepository } from './webhooks.repository';-import type { Webhook, WebhooksConfig } from './webhooks.types';+import type { Webhook, WebhookMultiplePayloads, WebhooksConfig } from './webhooks.types';import { triggerWebhook as triggerWebhookServiceImpl } from '@papra/webhooks';import pLimit from 'p-limit';import { createDeferable } from '../shared/async/defer';import { createLogger } from '../shared/logger/logger';-import { isUrlSsrfSafe } from '../shared/ssrf/ssrf.services';+import { createCachedIsUrlSsrfSafeFunction, isUrlSsrfSafe } from '../shared/ssrf/ssrf.services';import { WEBHOOK_URL_ALLOWED_HOSTNAMES_ENV_VAR } from './webhooks.constants';import { createSsrfUnsafeUrlError, createWebhookNotFoundError } from './webhooks.errors';@@ -105,7 +105,7 @@ export async function triggerWebhooks({logger = createLogger({ namespace: 'webhook' }),triggerWebhookService = triggerWebhookServiceImpl,webhooksConfig,- ...webhookData+ ...multiplePayloadsData}: {webhookRepository: WebhookRepository;organizationId: string;@@ -113,32 +113,79 @@ export async function triggerWebhooks({logger?: Logger;triggerWebhookService?: typeof triggerWebhookServiceImpl;webhooksConfig: WebhooksConfig;-} & WebhookPayloads) {- const { event } = webhookData;+} & WebhookMultiplePayloads) {+ const { event } = multiplePayloadsData;+ const singlePayloads = splitMultiplePayloads(multiplePayloadsData);+const { webhooks } = await webhookRepository.getOrganizationEnabledWebhooksForEvent({ organizationId, event });- logger.info({ webhooksCount: webhooks.length, organizationId, event }, 'Triggering webhooks');+ logger.info({ webhooksCount: webhooks.length, organizationId, event, payloadsCount: singlePayloads.length }, 'Triggering webhooks');++ const safeWebhooks = await filterOutSsrfUnsafeWebhooks({ webhooks, webhooksConfig, logger });const limit = pLimit(10);await Promise.all(- webhooks.map(async webhook =>- limit(async () =>- triggerWebhook({ webhook, webhookRepository, now, ...webhookData, logger, triggerWebhookService, webhooksConfig }),+ safeWebhooks.flatMap(webhook =>+ singlePayloads.map(async webhookData =>+ limit(async () =>+ triggerWebhook({ webhook, webhookRepository, now, ...webhookData, logger, triggerWebhookService }),+ ),),),);}+async function filterOutSsrfUnsafeWebhooks({+ webhooks,+ webhooksConfig,+ logger,+}: {+ webhooks: Webhook[];+ webhooksConfig: WebhooksConfig;+ logger: Logger;+}): Promise<Webhook[]> {+ if (!webhooksConfig.isSsrfProtectionEnabled) {+ return webhooks;+ }++ const limit = pLimit(10);++ // Caching is done at the function level to limit the TOCTOU window while still avoiding redundant checks for duplicate URLs across webhooks+ const isUrlSsrfSafeCached = createCachedIsUrlSsrfSafeFunction({+ allowedHostnames: webhooksConfig.webhookUrlAllowedHostnames,+ logger,+ });++ const checked = await Promise.all(+ webhooks.map(async webhook =>+ limit(async () => {+ const isSafe = await isUrlSsrfSafeCached({ url: webhook.url });++ if (!isSafe) {+ reportNonSsrfSafeWebhookUrl({ url: webhook.url, logger });+ }++ return { webhook, isSafe };+ }),+ ),+ );++ return checked.filter(({ isSafe }) => isSafe).map(({ webhook }) => webhook);+}++function splitMultiplePayloads(data: WebhookMultiplePayloads): WebhookPayloads[] {+ return data.payloads.map(payload => ({ event: data.event, payload })) as WebhookPayloads[];+}+export const deferTriggerWebhooks = createDeferable(triggerWebhooks);-export async function triggerWebhook({+async function triggerWebhook({webhook,webhookRepository,now = new Date(),logger = createLogger({ namespace: 'webhook' }),triggerWebhookService = triggerWebhookServiceImpl,- webhooksConfig,...webhookData}: {webhook: Webhook;@@ -146,21 +193,10 @@ export async function triggerWebhook({now?: Date;logger?: Logger;triggerWebhookService?: typeof triggerWebhookServiceImpl;- webhooksConfig: WebhooksConfig;} & WebhookPayloads) {const { url, secret, organizationId } = webhook;const { event } = webhookData;- // Check SSRF safety of the webhook URL before triggering the webhook- // still vulnerable to TOCTOU dns rebinbing attacks, but the timing window is really small- // and needs the attacker to have access to dns. It's currently an acceptable risk- await checkWebhookUrlIsSsrfSafe({- url,- isSsrfProtectionEnabled: webhooksConfig.isSsrfProtectionEnabled,- allowedHostnames: webhooksConfig.webhookUrlAllowedHostnames,- logger,- });-logger.info({ webhookId: webhook.id, event, organizationId }, 'Triggering webhook');const { responseData, responseStatus, requestPayload } = await triggerWebhookService({