Skip to content

The open alternative to Salesforce, designed for AI.

Import CalDAV events from servers that omit getetag (#25728)

neo773aea02a5aa080
diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant.ts
new file mode 100644
index 00000000000..9900815b7ee
--- /dev/null
+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant.ts
@@ -0,0 +1 @@
+export const CALDAV_UNVERSIONED_RESOURCE = 'unversioned';
diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.ts
index 93f6ca6da86..d60d6a649f6 100644
--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.ts
+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.ts
@@ -11,6 +11,7 @@ import {
} from 'tsdav';
import { isDefined } from 'twenty-shared/utils';
+import { CALDAV_UNVERSIONED_RESOURCE } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant';
import { type CalDavSyncCursor } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/types/caldav-sync-cursor';
import { extractICalData } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/extract-ical-data.util';
import { isCalDavCollectionHref } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/is-caldav-collection-href.util';
@@ -21,6 +22,7 @@ import { isSameCalDavResource } from 'src/modules/calendar/calendar-event-import
import { isValidCalDavHref } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/is-valid-caldav-href.util';
import { mapCalDavStatusToExceptionCode } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/map-caldav-status-to-exception-code.util';
import { parseICalEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/parse-ical-event.util';
+import { resolveCalDavResourceVersion } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/resolve-caldav-resource-version.util';
import { CalendarEventImportDriverException } from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception';
import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event';
@@ -314,18 +316,22 @@ export class CalDavFetchEventsService {
const currentEtags = await this.fetchEtagsByHref(client, calendar.url);
const changedHrefs = Object.keys(currentEtags).filter(
- (href) => storedEtags[href] !== currentEtags[href],
+ (href) =>
+ currentEtags[href] === CALDAV_UNVERSIONED_RESOURCE ||
+ storedEtags[href] !== currentEtags[href],
);
const cancelledHrefs = Object.keys(storedEtags).filter(
(href) =>
!(href in currentEtags) && !isCalDavCollectionHref(href, calendar.url),
);
+ const listedResources = Object.keys(currentEtags).length > 0;
+
return {
calendarUrl: calendar.url,
changedHrefs,
cancelledHrefs,
- newCtag,
+ newCtag: listedResources ? newCtag : undefined,
newEtags: currentEtags,
};
}
@@ -355,24 +361,25 @@ export class CalDavFetchEventsService {
): Promise<Record<string, string>> {
const responses = await client.propfind({
url: calendarUrl,
- props: { [`${DAVNamespaceShort.DAV}:getetag`]: {} },
+ props: {
+ [`${DAVNamespaceShort.DAV}:getetag`]: {},
+ [`${DAVNamespaceShort.DAV}:getlastmodified`]: {},
+ },
depth: '1',
});
- return responses.reduce<Record<string, string>>((map, response) => {
- const href = response.href;
- const etag = response.props?.getetag;
+ const memberResponses = responses.filter(
+ (response): response is DAVResponse & { href: string } =>
+ isNonEmptyString(response.href) &&
+ !isCalDavCollectionHref(response.href, calendarUrl),
+ );
- if (
- !isNonEmptyString(href) ||
- !isNonEmptyString(etag) ||
- !isValidCalDavHref(href) ||
- isCalDavCollectionHref(href, calendarUrl)
- ) {
+ return memberResponses.reduce<Record<string, string>>((map, response) => {
+ if (!isValidCalDavHref(response.href)) {
return map;
}
- map[href] = etag;
+ map[response.href] = resolveCalDavResourceVersion(response);
return map;
}, {});
diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/__tests__/resolve-caldav-resource-version.util.spec.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/__tests__/resolve-caldav-resource-version.util.spec.ts
new file mode 100644
index 00000000000..d1c9a5bcc2e
--- /dev/null
+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/__tests__/resolve-caldav-resource-version.util.spec.ts
@@ -0,0 +1,52 @@
+import { CALDAV_UNVERSIONED_RESOURCE } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant';
+import { resolveCalDavResourceVersion } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/resolve-caldav-resource-version.util';
+
+describe('resolveCalDavResourceVersion', () => {
+ it('uses the entity tag when the server supplies one', () => {
+ expect(
+ resolveCalDavResourceVersion({
+ props: {
+ getetag: '"fffff-abcd3"',
+ getlastmodified: 'Wed, 15 Nov 2023 10:00:00 GMT',
+ },
+ }),
+ ).toBe('"fffff-abcd3"');
+ });
+
+ it('stringifies an unquoted numeric entity tag', () => {
+ expect(resolveCalDavResourceVersion({ props: { getetag: 17964947 } })).toBe(
+ '17964947',
+ );
+ });
+
+ it('falls back to the last modified date when the entity tag is empty', () => {
+ expect(
+ resolveCalDavResourceVersion({
+ props: {
+ getetag: {},
+ getlastmodified: 'Wed, 15 Nov 2023 10:00:00 GMT',
+ },
+ }),
+ ).toBe('Wed, 15 Nov 2023 10:00:00 GMT');
+ });
+
+ it('falls back to the last modified date when the entity tag is missing', () => {
+ expect(
+ resolveCalDavResourceVersion({
+ props: { getlastmodified: 'Wed, 15 Nov 2023 10:00:00 GMT' },
+ }),
+ ).toBe('Wed, 15 Nov 2023 10:00:00 GMT');
+ });
+
+ it('marks the resource unversioned when the server supplies neither', () => {
+ expect(resolveCalDavResourceVersion({ props: { getetag: '' } })).toBe(
+ CALDAV_UNVERSIONED_RESOURCE,
+ );
+ });
+
+ it('marks the resource unversioned when no properties came back at all', () => {
+ expect(resolveCalDavResourceVersion({ props: undefined })).toBe(
+ CALDAV_UNVERSIONED_RESOURCE,
+ );
+ });
+});
diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/resolve-caldav-resource-version.util.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/resolve-caldav-resource-version.util.ts
new file mode 100644
index 00000000000..1a576bcefd3
--- /dev/null
+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/resolve-caldav-resource-version.util.ts
@@ -0,0 +1,23 @@
+import { isNonEmptyString, isNumber } from '@sniptt/guards';
+import { type DAVResponse } from 'tsdav';
+
+import { CALDAV_UNVERSIONED_RESOURCE } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant';
+
+const readTag = (value: unknown): string | undefined => {
+ if (isNonEmptyString(value)) {
+ return value;
+ }
+
+ if (isNumber(value) && Number.isFinite(value)) {
+ return String(value);
+ }
+
+ return undefined;
+};
+
+export const resolveCalDavResourceVersion = (
+ response: Pick<DAVResponse, 'props'>,
+): string =>
+ readTag(response.props?.getetag) ??
+ readTag(response.props?.getlastmodified) ??
+ CALDAV_UNVERSIONED_RESOURCE;
diff --git a/packages/twenty-server/test/integration/caldav/suites/caldav-events-import-without-etags.integration-spec.ts b/packages/twenty-server/test/integration/caldav/suites/caldav-events-import-without-etags.integration-spec.ts
new file mode 100644
index 00000000000..a9305eaea3c
--- /dev/null
+++ b/packages/twenty-server/test/integration/caldav/suites/caldav-events-import-without-etags.integration-spec.ts
@@ -0,0 +1,231 @@
+import { randomUUID } from 'node:crypto';
+
+import { isNonEmptyString } from '@sniptt/guards';
+
+import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';
+
+import { deleteConnectedAccount } from 'test/integration/metadata/suites/connected-account/utils/delete-connected-account.util';
+import { saveImapSmtpCaldavAccount } from 'test/integration/metadata/suites/connected-account/utils/save-imap-smtp-caldav-account.util';
+import { updateConfigVariable } from 'test/integration/twenty-config/utils/update-config-variable.util';
+import { findImportedCalendarEventTitles } from 'test/integration/utils/find-imported-records.util';
+import { getCoreRepository } from 'test/integration/utils/get-core-repository.util';
+import { runCalendarChannelEventsImport } from 'test/integration/utils/run-calendar-channel-events-import.util';
+import { runCalendarChannelListFetch } from 'test/integration/utils/run-calendar-channel-list-fetch.util';
+import {
+ type NonCompliantCalDavProxy,
+ startNonCompliantCalDavProxy,
+} from 'test/integration/utils/start-non-compliant-caldav-proxy.util';
+import {
+ type RadicaleServer,
+ startRadicaleContainer,
+} from 'test/integration/utils/start-radicale-container.util';
+
+const HANDLE = `caldav-no-etag-${randomUUID()}@acme.test`;
+const COLLECTION = 'personal';
+const PASSWORD = 'radicale-password';
+
+const authorizationHeader = `Basic ${Buffer.from(`${HANDLE}:${PASSWORD}`).toString('base64')}`;
+
+const icalEvent = ({ uid, summary }: { uid: string; summary: string }) =>
+ [
+ 'BEGIN:VCALENDAR',
+ 'VERSION:2.0',
+ 'PRODID:-//Twenty//CalDAV integration test//EN',
+ 'BEGIN:VEVENT',
+ `UID:${uid}`,
+ 'DTSTAMP:20231101T000000Z',
+ 'DTSTART:20231115T100000Z',
+ 'DTEND:20231115T110000Z',
+ `SUMMARY:${summary}`,
+ 'END:VEVENT',
+ 'END:VCALENDAR',
+ ].join('\r\n');
+
+describe('CalDAV calendar events import without entity tags (integration)', () => {
+ let radicale: RadicaleServer;
+ let proxy: NonCompliantCalDavProxy;
+ let connectedAccountId: string;
+ let calendarChannelId: string;
+
+ const collectionUrl = () =>
+ `http://${radicale.host}:${radicale.port}/${HANDLE}/${COLLECTION}/`;
+
+ const putEvent = async ({ uid, summary }: { uid: string; summary: string }) =>
+ fetch(`${collectionUrl()}${uid}.ics`, {
+ method: 'PUT',
+ headers: {
+ 'Content-Type': 'text/calendar; charset=utf-8',
+ Authorization: authorizationHeader,
+ },
+ body: icalEvent({ uid, summary }),
+ });
+
+ const syncCalendarChannel = async () => {
+ await runCalendarChannelListFetch(calendarChannelId);
+ await runCalendarChannelEventsImport(calendarChannelId);
+ };
+
+ beforeAll(async () => {
+ await updateConfigVariable({
+ input: { key: 'OUTBOUND_HTTP_SAFE_MODE_ENABLED', value: false },
+ });
+
+ radicale = await startRadicaleContainer({
+ username: HANDLE,
+ password: PASSWORD,
+ });
+
+ await fetch(collectionUrl(), {
+ method: 'MKCOL',
+ headers: {
+ 'Content-Type': 'application/xml; charset=utf-8',
+ Authorization: authorizationHeader,
+ },
+ body: `<?xml version="1.0" encoding="utf-8"?>
+ <mkcol xmlns="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
+ <set><prop>
+ <resourcetype><collection/><c:calendar/></resourcetype>
+ <displayname>Personal</displayname>
+ <c:supported-calendar-component-set><c:comp name="VEVENT"/></c:supported-calendar-component-set>
+ </prop></set>
+ </mkcol>`,
+ });
+
+ proxy = await startNonCompliantCalDavProxy({
+ targetHost: radicale.host,
+ targetPort: radicale.port,
+ });
+
+ const { data } = await saveImapSmtpCaldavAccount({
+ input: {
+ handle: HANDLE,
+ connectionParameters: {
+ CALDAV: {
+ host: `http://${proxy.host}:${proxy.port}`,
+ port: proxy.port,
+ username: HANDLE,
+ password: PASSWORD,
+ },
+ },
+ },
+ expectToFail: false,
+ });
+
+ connectedAccountId = data.connectedAccountId;
+
+ calendarChannelId = (
+ await getCoreRepository<CalendarChannelEntity>(
+ CalendarChannelEntity,
+ ).findOneByOrFail({ connectedAccountId })
+ ).id;
+ }, 300000);
+
+ afterAll(async () => {
+ await updateConfigVariable({
+ input: { key: 'OUTBOUND_HTTP_SAFE_MODE_ENABLED', value: true },
+ }).catch(() => undefined);
+
+ if (isNonEmptyString(connectedAccountId)) {
+ await deleteConnectedAccount({
+ id: connectedAccountId,
+ expectToFail: false,
+ }).catch(() => undefined);
+ }
+
+ await proxy?.stop().catch(() => undefined);
+ await radicale?.stop().catch(() => undefined);
+ });
+
+ afterEach(() => {
+ proxy.degradations.blankLastModified = false;
+ proxy.degradations.hideCollectionMembers = false;
+ });
+
+ it('imports an event when the server never returns an entity tag', async () => {
+ const summary = `CalDAV untagged event ${randomUUID()}`;
+
+ await putEvent({ uid: `caldav-event-${randomUUID()}`, summary });
+
+ await syncCalendarChannel();
+
+ expect(await findImportedCalendarEventTitles([summary])).toEqual([summary]);
+ }, 300000);
+
+ it('imports an event added after the collection tag advances', async () => {
+ const firstSummary = `CalDAV untagged event ${randomUUID()}`;
+ const secondSummary = `CalDAV untagged event ${randomUUID()}`;
+
+ await putEvent({
+ uid: `caldav-event-${randomUUID()}`,
+ summary: firstSummary,
+ });
+ await syncCalendarChannel();
+
+ await putEvent({
+ uid: `caldav-event-${randomUUID()}`,
+ summary: secondSummary,
+ });
+ await syncCalendarChannel();
+
+ expect(
+ await findImportedCalendarEventTitles([firstSummary, secondSummary]),
+ ).toEqual([firstSummary, secondSummary].sort());
+ }, 300000);
+
+ it('re-imports the collection after a listing comes back with no members', async () => {
+ const importedSummary = `CalDAV untagged event ${randomUUID()}`;
+ const addedSummary = `CalDAV untagged event ${randomUUID()}`;
+
+ await putEvent({
+ uid: `caldav-event-${randomUUID()}`,
+ summary: importedSummary,
+ });
+ await syncCalendarChannel();
+
+ expect(await findImportedCalendarEventTitles([importedSummary])).toEqual([
+ importedSummary,
+ ]);
+
+ proxy.degradations.hideCollectionMembers = true;
+
+ await putEvent({
+ uid: `caldav-event-${randomUUID()}`,
+ summary: addedSummary,
+ });
+ await syncCalendarChannel();
+
+ expect(
+ await findImportedCalendarEventTitles([importedSummary, addedSummary]),
+ ).toEqual([]);
+
+ proxy.degradations.hideCollectionMembers = false;
+
+ await syncCalendarChannel();
+
+ expect(
+ await findImportedCalendarEventTitles([importedSummary, addedSummary]),
+ ).toEqual([importedSummary, addedSummary].sort());
+ }, 300000);
+
+ it('imports an edit when the server returns neither entity tag nor last modified date', async () => {
+ const uid = `caldav-event-${randomUUID()}`;
+ const originalSummary = `CalDAV unversioned event ${randomUUID()}`;
+ const editedSummary = `CalDAV unversioned event ${randomUUID()}`;
+
+ proxy.degradations.blankLastModified = true;
+
+ await putEvent({ uid, summary: originalSummary });
+ await syncCalendarChannel();
+
+ expect(await findImportedCalendarEventTitles([originalSummary])).toEqual([
+ originalSummary,
+ ]);
+
+ await putEvent({ uid, summary: editedSummary });
+ await syncCalendarChannel();
+
+ expect(await findImportedCalendarEventTitles([editedSummary])).toEqual([
+ editedSummary,
+ ]);
+ }, 300000);
+});
diff --git a/packages/twenty-server/test/integration/utils/start-non-compliant-caldav-proxy.util.ts b/packages/twenty-server/test/integration/utils/start-non-compliant-caldav-proxy.util.ts
new file mode 100644
index 00000000000..d29ed92ea1f
--- /dev/null
+++ b/packages/twenty-server/test/integration/utils/start-non-compliant-caldav-proxy.util.ts
@@ -0,0 +1,159 @@
+import { once } from 'node:events';
+import { createServer, request, type IncomingMessage } from 'node:http';
+import { type AddressInfo } from 'node:net';
+import { brotliDecompressSync, gunzipSync, inflateSync } from 'node:zlib';
+
+const PROXY_HOST = '127.0.0.1';
+
+const ENTITY_TAG_PATTERN =
+ /<((?:[\w.-]+:)?)getetag(?:\s[^>]*)?>[\s\S]*?<\/\1getetag>/g;
+const LAST_MODIFIED_PATTERN =
+ /<((?:[\w.-]+:)?)getlastmodified(?:\s[^>]*)?>[\s\S]*?<\/\1getlastmodified>/g;
+const SUPPORTED_REPORT_PATTERN =
+ /<((?:[\w.-]+:)?)supported-report>[\s\S]*?<\/\1supported-report>/g;
+const RESPONSE_PATTERN = /<((?:[\w.-]+:)?)response>[\s\S]*?<\/\1response>/g;
+const HREF_PATTERN = /<(?:[\w.-]+:)?href>\s*([^<]*?)\s*<\/(?:[\w.-]+:)?href>/;
+
+export type NonCompliantCalDavProxy = {
+ host: string;
+ port: number;
+ degradations: {
+ blankLastModified: boolean;
+ hideCollectionMembers: boolean;
+ };
+ stop: () => Promise<void>;
+};
+
+const decodeBody = (body: Buffer, contentEncoding: string): Buffer => {
+ if (body.length === 0) {
+ return body;
+ }
+
+ switch (contentEncoding) {
+ case 'gzip':
+ return gunzipSync(body);
+ case 'deflate':
+ return inflateSync(body);
+ case 'br':
+ return brotliDecompressSync(body);
+ default:
+ return body;
+ }
+};
+
+const readBody = async (message: IncomingMessage): Promise<Buffer> => {
+ const chunks: Buffer[] = [];
+
+ for await (const chunk of message) {
+ chunks.push(chunk as Buffer);
+ }
+
+ return Buffer.concat(chunks);
+};
+
+const blankEntityTags = (body: string): string =>
+ body.replace(ENTITY_TAG_PATTERN, '<$1getetag />');
+
+const blankLastModified = (body: string): string =>
+ body.replace(LAST_MODIFIED_PATTERN, '<$1getlastmodified />');
+
+const dropSyncCollectionReport = (body: string): string =>
+ body.replace(SUPPORTED_REPORT_PATTERN, (report) =>
+ report.includes('sync-collection') ? '' : report,
+ );
+
+const dropMemberResponses = (body: string): string =>
+ body.replace(RESPONSE_PATTERN, (response) =>
+ HREF_PATTERN.exec(response)?.[1].endsWith('/') ? response : '',
+ );
+
+export const startNonCompliantCalDavProxy = async ({
+ targetHost,
+ targetPort,
+}: {
+ targetHost: string;
+ targetPort: number;
+}): Promise<NonCompliantCalDavProxy> => {
+ const degradations = {
+ blankLastModified: false,
+ hideCollectionMembers: false,
+ };
+
+ const server = createServer((incoming, outgoing) => {
+ void readBody(incoming)
+ .then((requestBody) => {
+ const upstream = request(
+ {
+ host: targetHost,
+ port: targetPort,
+ method: incoming.method,
+ path: incoming.url,
+ headers: {
+ ...incoming.headers,
+ host: `${targetHost}:${targetPort}`,
+ 'accept-encoding': 'identity',
+ },
+ },
+ (upstreamResponse) => {
+ void readBody(upstreamResponse)
+ .then((responseBody) => {
+ const headers = { ...upstreamResponse.headers };
+ const contentEncoding = String(
+ headers['content-encoding'] ?? '',
+ );
+
+ delete headers['content-length'];
+ delete headers['transfer-encoding'];
+ delete headers['content-encoding'];
+
+ const decodedBody = decodeBody(responseBody, contentEncoding);
+ const isXml = String(headers['content-type'] ?? '').includes(
+ 'xml',
+ );
+ const rewrites = [blankEntityTags, dropSyncCollectionReport];
+
+ if (degradations.blankLastModified) {
+ rewrites.push(blankLastModified);
+ }
+
+ if (
+ degradations.hideCollectionMembers &&
+ incoming.method === 'PROPFIND'
+ ) {
+ rewrites.push(dropMemberResponses);
+ }
+
+ outgoing.writeHead(upstreamResponse.statusCode ?? 502, headers);
+ outgoing.end(
+ isXml
+ ? rewrites.reduce(
+ (body, rewrite) => rewrite(body),
+ decodedBody.toString('utf8'),
+ )
+ : decodedBody,
+ );
+ })
+ .catch(() => outgoing.destroy());
+ },
+ );
+
+ upstream.on('error', () => outgoing.destroy());
+ upstream.end(requestBody);
+ })
+ .catch(() => outgoing.destroy());
+ });
+
+ server.listen(0, PROXY_HOST);
+ await once(server, 'listening');
+
+ return {
+ host: PROXY_HOST,
+ port: (server.address() as AddressInfo).port,
+ degradations,
+ stop: async () => {
+ server.closeAllConnections();
+ server.close();
+ await once(server, 'close');
+ },
+ };
+};