Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

feat(webhooks): support batching multiple payloads per event (#1068)

Corentin Thomassetdc78b5ed7d11
diff --git a/apps/papra-server/src/modules/documents/events/webhooks.document-created.ts b/apps/papra-server/src/modules/documents/events/webhooks.document-created.ts
index 26673c4f..f30713b6 100644
--- a/apps/papra-server/src/modules/documents/events/webhooks.document-created.ts
+++ b/apps/papra-server/src/modules/documents/events/webhooks.document-created.ts
@@ -15,13 +15,13 @@ export function registerTriggerWebhooksOnDocumentCreatedHandler({
await webhookTriggerServices.triggerWebhooks({
organizationId: document.organizationId,
event: 'document:created',
- payload: {
+ payloads: [{
documentId: document.id,
organizationId: document.organizationId,
name: document.name,
createdAt: document.createdAt,
updatedAt: document.updatedAt,
- },
+ }],
});
},
});
diff --git a/apps/papra-server/src/modules/documents/events/webhooks.document-updated.ts b/apps/papra-server/src/modules/documents/events/webhooks.document-updated.ts
index ae0bff0f..c339ec69 100644
--- a/apps/papra-server/src/modules/documents/events/webhooks.document-updated.ts
+++ b/apps/papra-server/src/modules/documents/events/webhooks.document-updated.ts
@@ -15,11 +15,11 @@ export function registerTriggerWebhooksOnDocumentUpdatedHandler({
await webhookTriggerServices.triggerWebhooks({
organizationId: document.organizationId,
event: 'document:updated',
- payload: {
+ payloads: [{
documentId: document.id,
organizationId: document.organizationId,
...changes,
- },
+ }],
});
},
});
diff --git a/apps/papra-server/src/modules/documents/events/webhooks.documents-trashed.ts b/apps/papra-server/src/modules/documents/events/webhooks.documents-trashed.ts
index 06451ddb..38caefc4 100644
--- a/apps/papra-server/src/modules/documents/events/webhooks.documents-trashed.ts
+++ b/apps/papra-server/src/modules/documents/events/webhooks.documents-trashed.ts
@@ -1,6 +1,5 @@
import type { EventServices } from '../../app/events/events.services';
import type { WebhookTriggerServices } from '../../webhooks/webhooks.trigger.services';
-import pLimit from 'p-limit';
export function registerTriggerWebhooksOnDocumentsTrashedHandler({
eventServices,
@@ -13,22 +12,14 @@ export function registerTriggerWebhooksOnDocumentsTrashedHandler({
eventName: 'documents.trashed',
handlerName: 'trigger-webhooks',
async handler({ documentIds, organizationId }) {
- const limit = pLimit(10);
-
- await Promise.all(
- documentIds.map(async documentId =>
- limit(async () =>
- webhookTriggerServices.triggerWebhooks({
- organizationId,
- event: 'document:deleted',
- payload: {
- documentId,
- organizationId,
- },
- }),
- ),
- ),
- );
+ await webhookTriggerServices.triggerWebhooks({
+ organizationId,
+ event: 'document:deleted',
+ payloads: documentIds.map(documentId => ({
+ documentId,
+ organizationId,
+ })),
+ });
},
});
}
diff --git a/apps/papra-server/src/modules/shared/ssrf/ssrf.services.test.ts b/apps/papra-server/src/modules/shared/ssrf/ssrf.services.test.ts
index 7d4affe3..e06899d1 100644
--- a/apps/papra-server/src/modules/shared/ssrf/ssrf.services.test.ts
+++ b/apps/papra-server/src/modules/shared/ssrf/ssrf.services.test.ts
@@ -1,6 +1,6 @@
import { describe, expect, test } from 'vitest';
import { createTestLogger } from '../logger/logger.test-utils';
-import { isUrlSsrfSafe } from './ssrf.services';
+import { createCachedIsUrlSsrfSafeFunction, isUrlSsrfSafe } from './ssrf.services';
describe('ssrf services', () => {
describe('isUrlSsrfSafe', () => {
@@ -103,8 +103,7 @@ describe('ssrf services', () => {
const logs = getLogs({ excludeTimestampMs: true });
expect(logs.length).toBe(1);
- expect(logs[0]).to.contain({ level: 'error', message: 'Error while checking if URL is SSRF safe' });
- expect(logs[0]?.data.error).to.contain({ message: 'Invalid URL', name: 'TypeError' });
+ expect(logs[0]).to.contain({ level: 'error', message: 'Invalid URL provided, cannot extract hostname' });
});
test('given an url with an hostname, if the DNS lookup throws an error, the function should return false and not throw', async () => {
@@ -118,9 +117,32 @@ describe('ssrf services', () => {
const logs = getLogs({ excludeTimestampMs: true });
expect(logs.length).toBe(1);
- expect(logs[0]).to.contain({ level: 'error', message: 'Error while checking if URL is SSRF safe' });
+ expect(logs[0]).to.contain({ level: 'error', message: 'Error while checking if hostname is SSRF safe' });
expect(logs[0]?.data.error).to.contain({ message: 'DNS lookup failed', name: 'Error' });
});
});
});
+
+ describe('createCachedIsUrlSsrfSafeFunction', () => {
+ describe(`when checking if multiple urls are SSRF safe, to avoid performing multiple DNS lookups for the same hostnames, a caching layer can be used
+ note that this should be used with caution as it can be vulnerable to TOCTOU DNS rebinding attacks in the window between the cache check and the actual request
+ so to be used for one-time checks of webhook urls for example, but not for every request to a given url`, () => {
+ test('when an hostname is checked multiple times, the DNS lookup should only be performed once', async () => {
+ let dnsLookupCallCount = 0;
+
+ const dnsLookup = async () => {
+ dnsLookupCallCount++;
+ return { addresses: [{ address: '8.8.8.8', family: 4 }] };
+ };
+
+ const cachedIsUrlSsrfSafe = createCachedIsUrlSsrfSafeFunction({ dnsLookup });
+
+ await cachedIsUrlSsrfSafe({ url: 'http://example.com/foo/bar' });
+ await cachedIsUrlSsrfSafe({ url: 'http://example.com/foo/bar' });
+ await cachedIsUrlSsrfSafe({ url: 'http://example.com/other/path' });
+
+ expect(dnsLookupCallCount).toBe(1);
+ });
+ });
+ });
});
diff --git a/apps/papra-server/src/modules/shared/ssrf/ssrf.services.ts b/apps/papra-server/src/modules/shared/ssrf/ssrf.services.ts
index 11731c10..a522f871 100644
--- a/apps/papra-server/src/modules/shared/ssrf/ssrf.services.ts
+++ b/apps/papra-server/src/modules/shared/ssrf/ssrf.services.ts
@@ -2,6 +2,8 @@ import type { Logger } from '@crowlog/logger';
import { isIP } from 'node:net';
import { dnsLookupHostname } from '../dns/dns.services';
import { createLogger } from '../logger/logger';
+import { getUrlHostname } from '../urls/urls.models';
+import { isNil } from '../utils';
import { isIpSsrfSafe } from './ssrf.models';
export async function isUrlSsrfSafe({
@@ -15,9 +17,28 @@ export async function isUrlSsrfSafe({
dnsLookup?: (args: { hostname: string }) => Promise<{ addresses: { address: string; family: number }[] }>;
logger?: Logger;
}): Promise<boolean> {
- try {
- const { hostname } = new URL(url);
+ const hostname = getUrlHostname(url);
+
+ if (isNil(hostname)) {
+ logger.error({ url }, 'Invalid URL provided, cannot extract hostname');
+ return false;
+ }
+ return isHostnameSsrfSafe({ hostname, allowedHostnames, dnsLookup, logger });
+}
+
+async function isHostnameSsrfSafe({
+ hostname,
+ allowedHostnames,
+ dnsLookup = dnsLookupHostname,
+ logger = createLogger({ namespace: 'ssrf.services' }),
+}: {
+ hostname: string;
+ allowedHostnames?: Set<string>;
+ dnsLookup?: (args: { hostname: string }) => Promise<{ addresses: { address: string; family: number }[] }>;
+ logger?: Logger;
+}): Promise<boolean> {
+ try {
if (allowedHostnames && allowedHostnames.has(hostname)) {
return true;
}
@@ -30,7 +51,40 @@ export async function isUrlSsrfSafe({
return addresses.length > 0 && addresses.every(({ address }) => isIpSsrfSafe(address));
} catch (error) {
- logger.error({ error, url }, 'Error while checking if URL is SSRF safe');
+ logger.error({ error, hostname }, 'Error while checking if hostname is SSRF safe');
return false;
}
}
+
+export function createCachedIsUrlSsrfSafeFunction({
+ allowedHostnames,
+ dnsLookup,
+ logger = createLogger({ namespace: 'ssrf.services' }),
+}: {
+ allowedHostnames?: Set<string>;
+ dnsLookup?: (args: { hostname: string }) => Promise<{ addresses: { address: string; family: number }[] }>;
+ logger?: Logger;
+}) {
+ const hostnameCache = new Map<string, boolean>();
+
+ return async ({ url }: { url: string }) => {
+ const hostname = getUrlHostname(url);
+
+ if (isNil(hostname)) {
+ logger.error({ url }, 'Invalid URL provided, cannot extract hostname');
+ return false;
+ }
+
+ const cachedResult = hostnameCache.get(hostname);
+
+ if (cachedResult !== undefined) {
+ return cachedResult;
+ }
+
+ const isSafe = await isHostnameSsrfSafe({ hostname, allowedHostnames, dnsLookup, logger });
+
+ hostnameCache.set(hostname, isSafe);
+
+ return isSafe;
+ };
+}
diff --git a/apps/papra-server/src/modules/shared/urls/urls.models.test.ts b/apps/papra-server/src/modules/shared/urls/urls.models.test.ts
new file mode 100644
index 00000000..4c5c6ea1
--- /dev/null
+++ b/apps/papra-server/src/modules/shared/urls/urls.models.test.ts
@@ -0,0 +1,23 @@
+import { describe, expect, test } from 'vitest';
+import { getUrlHostname } from './urls.models';
+
+describe('urls.models', () => {
+ describe('getUrlHostname', () => {
+ test('safely get the hostname of a url', () => {
+ expect(getUrlHostname('http://example.com/foo/bar')).to.eql('example.com');
+ expect(getUrlHostname('https://example.com/foo/bar')).to.eql('example.com');
+ expect(getUrlHostname('ftp://example.com/foo/bar')).to.eql('example.com');
+ expect(getUrlHostname('http://example.com:8080/foo/bar')).to.eql('example.com');
+ expect(getUrlHostname('http://user:[email protected]/foo/bar')).to.eql('example.com');
+ expect(getUrlHostname('http://127.0.0.1/foo/bar')).to.eql('127.0.0.1');
+ expect(getUrlHostname('http://[2001:db8::1]/foo/bar')).to.eql('[2001:db8::1]');
+ });
+
+ test('given an invalid url, return null', () => {
+ expect(getUrlHostname('not a url')).to.eql(null);
+ expect(getUrlHostname('http://')).to.eql(null);
+ expect(getUrlHostname('foo/bar')).to.eql(null);
+ expect(getUrlHostname('')).to.eql(null);
+ });
+ });
+});
diff --git a/apps/papra-server/src/modules/shared/urls/urls.models.ts b/apps/papra-server/src/modules/shared/urls/urls.models.ts
new file mode 100644
index 00000000..b3c03316
--- /dev/null
+++ b/apps/papra-server/src/modules/shared/urls/urls.models.ts
@@ -0,0 +1,7 @@
+export function getUrlHostname(url: string) {
+ try {
+ return new URL(url).hostname;
+ } catch {
+ return null;
+ }
+}
diff --git a/apps/papra-server/src/modules/tags/tags.routes.ts b/apps/papra-server/src/modules/tags/tags.routes.ts
index 0d255f82..c12b31e4 100644
--- a/apps/papra-server/src/modules/tags/tags.routes.ts
+++ b/apps/papra-server/src/modules/tags/tags.routes.ts
@@ -234,7 +234,7 @@ function setupRemoveTagFromDocumentRoute({ app, db, webhookTriggerServices }: Ro
webhookTriggerServices.deferTriggerWebhooks({
organizationId,
event: 'document:tag:removed',
- payload: { documentId, organizationId, tagId, tagName: tag.name },
+ payloads: [{ documentId, organizationId, tagId, tagName: tag.name }],
});
deferRegisterDocumentActivityLog({
diff --git a/apps/papra-server/src/modules/tags/tags.usecases.ts b/apps/papra-server/src/modules/tags/tags.usecases.ts
index 6324f192..d2cf0a8a 100644
--- a/apps/papra-server/src/modules/tags/tags.usecases.ts
+++ b/apps/papra-server/src/modules/tags/tags.usecases.ts
@@ -70,7 +70,7 @@ export async function addTagToDocument({
webhookTriggerServices.deferTriggerWebhooks({
organizationId,
event: 'document:tag:added',
- payload: { documentId, organizationId, tagId, tagName: tag.name },
+ payloads: [{ documentId, organizationId, tagId, tagName: tag.name }],
});
deferRegisterDocumentActivityLog({
diff --git a/apps/papra-server/src/modules/webhooks/webhooks.types.ts b/apps/papra-server/src/modules/webhooks/webhooks.types.ts
index 4007ccc1..ef7de5b5 100644
--- a/apps/papra-server/src/modules/webhooks/webhooks.types.ts
+++ b/apps/papra-server/src/modules/webhooks/webhooks.types.ts
@@ -1,3 +1,4 @@
+import type { WebhookPayloads } from '@papra/webhooks';
import type { Config } from '../config/config.types';
import type { webhookDeliveriesTable, webhookEventsTable, webhooksTable } from './webhooks.tables';
@@ -7,3 +8,10 @@ export type WebhookDelivery = typeof webhookDeliveriesTable.$inferSelect;
export type WebhookDeliveryInsert = typeof webhookDeliveriesTable.$inferInsert;
export type WebhooksConfig = Config['webhooks'];
+
+export type WebhookMultiplePayloads = {
+ [E in WebhookPayloads['event']]: {
+ event: E;
+ payloads: Extract<WebhookPayloads, { event: E }>['payload'][];
+ };
+}[WebhookPayloads['event']];
diff --git a/apps/papra-server/src/modules/webhooks/webhooks.usecases.test.ts b/apps/papra-server/src/modules/webhooks/webhooks.usecases.test.ts
index 66e3cc8a..04d8fc3d 100644
--- a/apps/papra-server/src/modules/webhooks/webhooks.usecases.test.ts
+++ b/apps/papra-server/src/modules/webhooks/webhooks.usecases.test.ts
@@ -164,16 +164,18 @@ describe('webhook usecases', () => {
});
describe('triggerWebhooks', () => {
- test('when SSRF protection is enabled, triggering a webhook with a private IP URL does not call the webhook service', async () => {
+ test('when SSRF protection is enabled, webhooks with unsafe URLs are filtered out and others still receive the event', async () => {
const { db } = await createInMemoryDatabase({
organizations: [
{ id: 'org_1', name: 'Organization 1' },
],
webhooks: [
- { id: 'wbh_1', name: 'Test Webhook', url: 'https://192.168.1.1/webhook', organizationId: 'org_1', enabled: true },
+ { id: 'wbh_1', name: 'Unsafe Webhook', url: 'https://192.168.1.1/webhook', organizationId: 'org_1', enabled: true },
+ { id: 'wbh_2', name: 'Safe Webhook', url: 'https://example.com/webhook', organizationId: 'org_1', enabled: true },
],
webhookEvents: [
{ id: 'wbh_ev_1', webhookId: 'wbh_1', eventName: 'document:created' },
+ { id: 'wbh_ev_2', webhookId: 'wbh_2', eventName: 'document:created' },
],
});
@@ -181,17 +183,17 @@ describe('webhook usecases', () => {
const { logger } = createTestLogger();
const triggerWebhookServiceArgs: unknown[] = [];
- await expect(triggerWebhooks({
+ await triggerWebhooks({
webhookRepository,
organizationId: 'org_1',
event: 'document:created',
- payload: {
+ payloads: [{
documentId: 'doc_1',
organizationId: 'org_1',
name: 'Document 1',
createdAt: new Date('2025-01-01'),
updatedAt: new Date('2025-01-01'),
- },
+ }],
logger,
triggerWebhookService: async (args) => {
triggerWebhookServiceArgs.push(args);
@@ -201,9 +203,10 @@ describe('webhook usecases', () => {
isSsrfProtectionEnabled: true,
webhookUrlAllowedHostnames: new Set<string>(),
},
- })).rejects.toThrow('The provided URL is not safe to perform requests to');
+ });
- expect(triggerWebhookServiceArgs).to.eql([]);
+ expect(triggerWebhookServiceArgs).to.have.lengthOf(1);
+ expect((triggerWebhookServiceArgs[0] as { webhookUrl: string }).webhookUrl).to.eql('https://example.com/webhook');
});
test('when an organization has webhooks enabled for an event, the configured urls are called with the event payload', async () => {
@@ -249,7 +252,7 @@ describe('webhook usecases', () => {
webhookRepository,
organizationId: 'org_1',
event: 'document:created',
- payload: eventPayload,
+ payloads: [eventPayload],
logger,
now: new Date('2025-05-04'),
triggerWebhookService: async (args) => {
diff --git a/apps/papra-server/src/modules/webhooks/webhooks.usecases.ts b/apps/papra-server/src/modules/webhooks/webhooks.usecases.ts
index fc719139..c170caf9 100644
--- a/apps/papra-server/src/modules/webhooks/webhooks.usecases.ts
+++ b/apps/papra-server/src/modules/webhooks/webhooks.usecases.ts
@@ -1,12 +1,12 @@
import type { EventName, WebhookPayloads } from '@papra/webhooks';
import type { Logger } from '../shared/logger/logger';
import type { WebhookRepository } from './webhooks.repository';
-import type { Webhook, WebhooksConfig } from './webhooks.types';
+import type { Webhook, WebhookMultiplePayloads, WebhooksConfig } from './webhooks.types';
import { triggerWebhook as triggerWebhookServiceImpl } from '@papra/webhooks';
import pLimit from 'p-limit';
import { createDeferable } from '../shared/async/defer';
import { createLogger } from '../shared/logger/logger';
-import { isUrlSsrfSafe } from '../shared/ssrf/ssrf.services';
+import { createCachedIsUrlSsrfSafeFunction, isUrlSsrfSafe } from '../shared/ssrf/ssrf.services';
import { WEBHOOK_URL_ALLOWED_HOSTNAMES_ENV_VAR } from './webhooks.constants';
import { createSsrfUnsafeUrlError, createWebhookNotFoundError } from './webhooks.errors';
@@ -105,7 +105,7 @@ export async function triggerWebhooks({
logger = createLogger({ namespace: 'webhook' }),
triggerWebhookService = triggerWebhookServiceImpl,
webhooksConfig,
- ...webhookData
+ ...multiplePayloadsData
}: {
webhookRepository: WebhookRepository;
organizationId: string;
@@ -113,32 +113,79 @@ export async function triggerWebhooks({
logger?: Logger;
triggerWebhookService?: typeof triggerWebhookServiceImpl;
webhooksConfig: WebhooksConfig;
-} & WebhookPayloads) {
- const { event } = webhookData;
+} & WebhookMultiplePayloads) {
+ const { event } = multiplePayloadsData;
+ const singlePayloads = splitMultiplePayloads(multiplePayloadsData);
+
const { webhooks } = await webhookRepository.getOrganizationEnabledWebhooksForEvent({ organizationId, event });
- logger.info({ webhooksCount: webhooks.length, organizationId, event }, 'Triggering webhooks');
+ logger.info({ webhooksCount: webhooks.length, organizationId, event, payloadsCount: singlePayloads.length }, 'Triggering webhooks');
+
+ const safeWebhooks = await filterOutSsrfUnsafeWebhooks({ webhooks, webhooksConfig, logger });
const limit = pLimit(10);
await Promise.all(
- webhooks.map(async webhook =>
- limit(async () =>
- triggerWebhook({ webhook, webhookRepository, now, ...webhookData, logger, triggerWebhookService, webhooksConfig }),
+ safeWebhooks.flatMap(webhook =>
+ singlePayloads.map(async webhookData =>
+ limit(async () =>
+ triggerWebhook({ webhook, webhookRepository, now, ...webhookData, logger, triggerWebhookService }),
+ ),
),
),
);
}
+async function filterOutSsrfUnsafeWebhooks({
+ webhooks,
+ webhooksConfig,
+ logger,
+}: {
+ webhooks: Webhook[];
+ webhooksConfig: WebhooksConfig;
+ logger: Logger;
+}): Promise<Webhook[]> {
+ if (!webhooksConfig.isSsrfProtectionEnabled) {
+ return webhooks;
+ }
+
+ const limit = pLimit(10);
+
+ // Caching is done at the function level to limit the TOCTOU window while still avoiding redundant checks for duplicate URLs across webhooks
+ const isUrlSsrfSafeCached = createCachedIsUrlSsrfSafeFunction({
+ allowedHostnames: webhooksConfig.webhookUrlAllowedHostnames,
+ logger,
+ });
+
+ const checked = await Promise.all(
+ webhooks.map(async webhook =>
+ limit(async () => {
+ const isSafe = await isUrlSsrfSafeCached({ url: webhook.url });
+
+ if (!isSafe) {
+ reportNonSsrfSafeWebhookUrl({ url: webhook.url, logger });
+ }
+
+ return { webhook, isSafe };
+ }),
+ ),
+ );
+
+ return checked.filter(({ isSafe }) => isSafe).map(({ webhook }) => webhook);
+}
+
+function splitMultiplePayloads(data: WebhookMultiplePayloads): WebhookPayloads[] {
+ return data.payloads.map(payload => ({ event: data.event, payload })) as WebhookPayloads[];
+}
+
export const deferTriggerWebhooks = createDeferable(triggerWebhooks);
-export async function triggerWebhook({
+async function triggerWebhook({
webhook,
webhookRepository,
now = new Date(),
logger = createLogger({ namespace: 'webhook' }),
triggerWebhookService = triggerWebhookServiceImpl,
- webhooksConfig,
...webhookData
}: {
webhook: Webhook;
@@ -146,21 +193,10 @@ export async function triggerWebhook({
now?: Date;
logger?: Logger;
triggerWebhookService?: typeof triggerWebhookServiceImpl;
- webhooksConfig: WebhooksConfig;
} & WebhookPayloads) {
const { url, secret, organizationId } = webhook;
const { event } = webhookData;
- // Check SSRF safety of the webhook URL before triggering the webhook
- // still vulnerable to TOCTOU dns rebinbing attacks, but the timing window is really small
- // and needs the attacker to have access to dns. It's currently an acceptable risk
- await checkWebhookUrlIsSsrfSafe({
- url,
- isSsrfProtectionEnabled: webhooksConfig.isSsrfProtectionEnabled,
- allowedHostnames: webhooksConfig.webhookUrlAllowedHostnames,
- logger,
- });
-
logger.info({ webhookId: webhook.id, event, organizationId }, 'Triggering webhook');
const { responseData, responseStatus, requestPayload } = await triggerWebhookService({