The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Import CalDAV events from servers that omit getetag (#25728)
neo773aea02a5aa080
diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant.tsnew file mode 100644index 00000000000..9900815b7ee--- /dev/null+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant.ts@@ -0,0 +1 @@+export const CALDAV_UNVERSIONED_RESOURCE = 'unversioned';diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.tsindex 93f6ca6da86..d60d6a649f6 100644--- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.ts+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/services/caldav-fetch-events.service.ts@@ -11,6 +11,7 @@ import {} from 'tsdav';import { isDefined } from 'twenty-shared/utils';+import { CALDAV_UNVERSIONED_RESOURCE } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant';import { type CalDavSyncCursor } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/types/caldav-sync-cursor';import { extractICalData } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/extract-ical-data.util';import { isCalDavCollectionHref } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/is-caldav-collection-href.util';@@ -21,6 +22,7 @@ import { isSameCalDavResource } from 'src/modules/calendar/calendar-event-importimport { isValidCalDavHref } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/is-valid-caldav-href.util';import { mapCalDavStatusToExceptionCode } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/map-caldav-status-to-exception-code.util';import { parseICalEvents } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/parse-ical-event.util';+import { resolveCalDavResourceVersion } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/resolve-caldav-resource-version.util';import { CalendarEventImportDriverException } from 'src/modules/calendar/calendar-event-import-manager/drivers/exceptions/calendar-event-import-driver.exception';import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event';@@ -314,18 +316,22 @@ export class CalDavFetchEventsService {const currentEtags = await this.fetchEtagsByHref(client, calendar.url);const changedHrefs = Object.keys(currentEtags).filter(- (href) => storedEtags[href] !== currentEtags[href],+ (href) =>+ currentEtags[href] === CALDAV_UNVERSIONED_RESOURCE ||+ storedEtags[href] !== currentEtags[href],);const cancelledHrefs = Object.keys(storedEtags).filter((href) =>!(href in currentEtags) && !isCalDavCollectionHref(href, calendar.url),);+ const listedResources = Object.keys(currentEtags).length > 0;+return {calendarUrl: calendar.url,changedHrefs,cancelledHrefs,- newCtag,+ newCtag: listedResources ? newCtag : undefined,newEtags: currentEtags,};}@@ -355,24 +361,25 @@ export class CalDavFetchEventsService {): Promise<Record<string, string>> {const responses = await client.propfind({url: calendarUrl,- props: { [`${DAVNamespaceShort.DAV}:getetag`]: {} },+ props: {+ [`${DAVNamespaceShort.DAV}:getetag`]: {},+ [`${DAVNamespaceShort.DAV}:getlastmodified`]: {},+ },depth: '1',});- return responses.reduce<Record<string, string>>((map, response) => {- const href = response.href;- const etag = response.props?.getetag;+ const memberResponses = responses.filter(+ (response): response is DAVResponse & { href: string } =>+ isNonEmptyString(response.href) &&+ !isCalDavCollectionHref(response.href, calendarUrl),+ );- if (- !isNonEmptyString(href) ||- !isNonEmptyString(etag) ||- !isValidCalDavHref(href) ||- isCalDavCollectionHref(href, calendarUrl)- ) {+ return memberResponses.reduce<Record<string, string>>((map, response) => {+ if (!isValidCalDavHref(response.href)) {return map;}- map[href] = etag;+ map[response.href] = resolveCalDavResourceVersion(response);return map;}, {});diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/__tests__/resolve-caldav-resource-version.util.spec.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/__tests__/resolve-caldav-resource-version.util.spec.tsnew file mode 100644index 00000000000..d1c9a5bcc2e--- /dev/null+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/__tests__/resolve-caldav-resource-version.util.spec.ts@@ -0,0 +1,52 @@+import { CALDAV_UNVERSIONED_RESOURCE } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant';+import { resolveCalDavResourceVersion } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/resolve-caldav-resource-version.util';++describe('resolveCalDavResourceVersion', () => {+ it('uses the entity tag when the server supplies one', () => {+ expect(+ resolveCalDavResourceVersion({+ props: {+ getetag: '"fffff-abcd3"',+ getlastmodified: 'Wed, 15 Nov 2023 10:00:00 GMT',+ },+ }),+ ).toBe('"fffff-abcd3"');+ });++ it('stringifies an unquoted numeric entity tag', () => {+ expect(resolveCalDavResourceVersion({ props: { getetag: 17964947 } })).toBe(+ '17964947',+ );+ });++ it('falls back to the last modified date when the entity tag is empty', () => {+ expect(+ resolveCalDavResourceVersion({+ props: {+ getetag: {},+ getlastmodified: 'Wed, 15 Nov 2023 10:00:00 GMT',+ },+ }),+ ).toBe('Wed, 15 Nov 2023 10:00:00 GMT');+ });++ it('falls back to the last modified date when the entity tag is missing', () => {+ expect(+ resolveCalDavResourceVersion({+ props: { getlastmodified: 'Wed, 15 Nov 2023 10:00:00 GMT' },+ }),+ ).toBe('Wed, 15 Nov 2023 10:00:00 GMT');+ });++ it('marks the resource unversioned when the server supplies neither', () => {+ expect(resolveCalDavResourceVersion({ props: { getetag: '' } })).toBe(+ CALDAV_UNVERSIONED_RESOURCE,+ );+ });++ it('marks the resource unversioned when no properties came back at all', () => {+ expect(resolveCalDavResourceVersion({ props: undefined })).toBe(+ CALDAV_UNVERSIONED_RESOURCE,+ );+ });+});diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/resolve-caldav-resource-version.util.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/resolve-caldav-resource-version.util.tsnew file mode 100644index 00000000000..1a576bcefd3--- /dev/null+++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/drivers/caldav/utils/resolve-caldav-resource-version.util.ts@@ -0,0 +1,23 @@+import { isNonEmptyString, isNumber } from '@sniptt/guards';+import { type DAVResponse } from 'tsdav';++import { CALDAV_UNVERSIONED_RESOURCE } from 'src/modules/calendar/calendar-event-import-manager/drivers/caldav/constants/caldav-unversioned-resource.constant';++const readTag = (value: unknown): string | undefined => {+ if (isNonEmptyString(value)) {+ return value;+ }++ if (isNumber(value) && Number.isFinite(value)) {+ return String(value);+ }++ return undefined;+};++export const resolveCalDavResourceVersion = (+ response: Pick<DAVResponse, 'props'>,+): string =>+ readTag(response.props?.getetag) ??+ readTag(response.props?.getlastmodified) ??+ CALDAV_UNVERSIONED_RESOURCE;diff --git a/packages/twenty-server/test/integration/caldav/suites/caldav-events-import-without-etags.integration-spec.ts b/packages/twenty-server/test/integration/caldav/suites/caldav-events-import-without-etags.integration-spec.tsnew file mode 100644index 00000000000..a9305eaea3c--- /dev/null+++ b/packages/twenty-server/test/integration/caldav/suites/caldav-events-import-without-etags.integration-spec.ts@@ -0,0 +1,231 @@+import { randomUUID } from 'node:crypto';++import { isNonEmptyString } from '@sniptt/guards';++import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';++import { deleteConnectedAccount } from 'test/integration/metadata/suites/connected-account/utils/delete-connected-account.util';+import { saveImapSmtpCaldavAccount } from 'test/integration/metadata/suites/connected-account/utils/save-imap-smtp-caldav-account.util';+import { updateConfigVariable } from 'test/integration/twenty-config/utils/update-config-variable.util';+import { findImportedCalendarEventTitles } from 'test/integration/utils/find-imported-records.util';+import { getCoreRepository } from 'test/integration/utils/get-core-repository.util';+import { runCalendarChannelEventsImport } from 'test/integration/utils/run-calendar-channel-events-import.util';+import { runCalendarChannelListFetch } from 'test/integration/utils/run-calendar-channel-list-fetch.util';+import {+ type NonCompliantCalDavProxy,+ startNonCompliantCalDavProxy,+} from 'test/integration/utils/start-non-compliant-caldav-proxy.util';+import {+ type RadicaleServer,+ startRadicaleContainer,+} from 'test/integration/utils/start-radicale-container.util';++const HANDLE = `caldav-no-etag-${randomUUID()}@acme.test`;+const COLLECTION = 'personal';+const PASSWORD = 'radicale-password';++const authorizationHeader = `Basic ${Buffer.from(`${HANDLE}:${PASSWORD}`).toString('base64')}`;++const icalEvent = ({ uid, summary }: { uid: string; summary: string }) =>+ [+ 'BEGIN:VCALENDAR',+ 'VERSION:2.0',+ 'PRODID:-//Twenty//CalDAV integration test//EN',+ 'BEGIN:VEVENT',+ `UID:${uid}`,+ 'DTSTAMP:20231101T000000Z',+ 'DTSTART:20231115T100000Z',+ 'DTEND:20231115T110000Z',+ `SUMMARY:${summary}`,+ 'END:VEVENT',+ 'END:VCALENDAR',+ ].join('\r\n');++describe('CalDAV calendar events import without entity tags (integration)', () => {+ let radicale: RadicaleServer;+ let proxy: NonCompliantCalDavProxy;+ let connectedAccountId: string;+ let calendarChannelId: string;++ const collectionUrl = () =>+ `http://${radicale.host}:${radicale.port}/${HANDLE}/${COLLECTION}/`;++ const putEvent = async ({ uid, summary }: { uid: string; summary: string }) =>+ fetch(`${collectionUrl()}${uid}.ics`, {+ method: 'PUT',+ headers: {+ 'Content-Type': 'text/calendar; charset=utf-8',+ Authorization: authorizationHeader,+ },+ body: icalEvent({ uid, summary }),+ });++ const syncCalendarChannel = async () => {+ await runCalendarChannelListFetch(calendarChannelId);+ await runCalendarChannelEventsImport(calendarChannelId);+ };++ beforeAll(async () => {+ await updateConfigVariable({+ input: { key: 'OUTBOUND_HTTP_SAFE_MODE_ENABLED', value: false },+ });++ radicale = await startRadicaleContainer({+ username: HANDLE,+ password: PASSWORD,+ });++ await fetch(collectionUrl(), {+ method: 'MKCOL',+ headers: {+ 'Content-Type': 'application/xml; charset=utf-8',+ Authorization: authorizationHeader,+ },+ body: `<?xml version="1.0" encoding="utf-8"?>+ <mkcol xmlns="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">+ <set><prop>+ <resourcetype><collection/><c:calendar/></resourcetype>+ <displayname>Personal</displayname>+ <c:supported-calendar-component-set><c:comp name="VEVENT"/></c:supported-calendar-component-set>+ </prop></set>+ </mkcol>`,+ });++ proxy = await startNonCompliantCalDavProxy({+ targetHost: radicale.host,+ targetPort: radicale.port,+ });++ const { data } = await saveImapSmtpCaldavAccount({+ input: {+ handle: HANDLE,+ connectionParameters: {+ CALDAV: {+ host: `http://${proxy.host}:${proxy.port}`,+ port: proxy.port,+ username: HANDLE,+ password: PASSWORD,+ },+ },+ },+ expectToFail: false,+ });++ connectedAccountId = data.connectedAccountId;++ calendarChannelId = (+ await getCoreRepository<CalendarChannelEntity>(+ CalendarChannelEntity,+ ).findOneByOrFail({ connectedAccountId })+ ).id;+ }, 300000);++ afterAll(async () => {+ await updateConfigVariable({+ input: { key: 'OUTBOUND_HTTP_SAFE_MODE_ENABLED', value: true },+ }).catch(() => undefined);++ if (isNonEmptyString(connectedAccountId)) {+ await deleteConnectedAccount({+ id: connectedAccountId,+ expectToFail: false,+ }).catch(() => undefined);+ }++ await proxy?.stop().catch(() => undefined);+ await radicale?.stop().catch(() => undefined);+ });++ afterEach(() => {+ proxy.degradations.blankLastModified = false;+ proxy.degradations.hideCollectionMembers = false;+ });++ it('imports an event when the server never returns an entity tag', async () => {+ const summary = `CalDAV untagged event ${randomUUID()}`;++ await putEvent({ uid: `caldav-event-${randomUUID()}`, summary });++ await syncCalendarChannel();++ expect(await findImportedCalendarEventTitles([summary])).toEqual([summary]);+ }, 300000);++ it('imports an event added after the collection tag advances', async () => {+ const firstSummary = `CalDAV untagged event ${randomUUID()}`;+ const secondSummary = `CalDAV untagged event ${randomUUID()}`;++ await putEvent({+ uid: `caldav-event-${randomUUID()}`,+ summary: firstSummary,+ });+ await syncCalendarChannel();++ await putEvent({+ uid: `caldav-event-${randomUUID()}`,+ summary: secondSummary,+ });+ await syncCalendarChannel();++ expect(+ await findImportedCalendarEventTitles([firstSummary, secondSummary]),+ ).toEqual([firstSummary, secondSummary].sort());+ }, 300000);++ it('re-imports the collection after a listing comes back with no members', async () => {+ const importedSummary = `CalDAV untagged event ${randomUUID()}`;+ const addedSummary = `CalDAV untagged event ${randomUUID()}`;++ await putEvent({+ uid: `caldav-event-${randomUUID()}`,+ summary: importedSummary,+ });+ await syncCalendarChannel();++ expect(await findImportedCalendarEventTitles([importedSummary])).toEqual([+ importedSummary,+ ]);++ proxy.degradations.hideCollectionMembers = true;++ await putEvent({+ uid: `caldav-event-${randomUUID()}`,+ summary: addedSummary,+ });+ await syncCalendarChannel();++ expect(+ await findImportedCalendarEventTitles([importedSummary, addedSummary]),+ ).toEqual([]);++ proxy.degradations.hideCollectionMembers = false;++ await syncCalendarChannel();++ expect(+ await findImportedCalendarEventTitles([importedSummary, addedSummary]),+ ).toEqual([importedSummary, addedSummary].sort());+ }, 300000);++ it('imports an edit when the server returns neither entity tag nor last modified date', async () => {+ const uid = `caldav-event-${randomUUID()}`;+ const originalSummary = `CalDAV unversioned event ${randomUUID()}`;+ const editedSummary = `CalDAV unversioned event ${randomUUID()}`;++ proxy.degradations.blankLastModified = true;++ await putEvent({ uid, summary: originalSummary });+ await syncCalendarChannel();++ expect(await findImportedCalendarEventTitles([originalSummary])).toEqual([+ originalSummary,+ ]);++ await putEvent({ uid, summary: editedSummary });+ await syncCalendarChannel();++ expect(await findImportedCalendarEventTitles([editedSummary])).toEqual([+ editedSummary,+ ]);+ }, 300000);+});diff --git a/packages/twenty-server/test/integration/utils/start-non-compliant-caldav-proxy.util.ts b/packages/twenty-server/test/integration/utils/start-non-compliant-caldav-proxy.util.tsnew file mode 100644index 00000000000..d29ed92ea1f--- /dev/null+++ b/packages/twenty-server/test/integration/utils/start-non-compliant-caldav-proxy.util.ts@@ -0,0 +1,159 @@+import { once } from 'node:events';+import { createServer, request, type IncomingMessage } from 'node:http';+import { type AddressInfo } from 'node:net';+import { brotliDecompressSync, gunzipSync, inflateSync } from 'node:zlib';++const PROXY_HOST = '127.0.0.1';++const ENTITY_TAG_PATTERN =+ /<((?:[\w.-]+:)?)getetag(?:\s[^>]*)?>[\s\S]*?<\/\1getetag>/g;+const LAST_MODIFIED_PATTERN =+ /<((?:[\w.-]+:)?)getlastmodified(?:\s[^>]*)?>[\s\S]*?<\/\1getlastmodified>/g;+const SUPPORTED_REPORT_PATTERN =+ /<((?:[\w.-]+:)?)supported-report>[\s\S]*?<\/\1supported-report>/g;+const RESPONSE_PATTERN = /<((?:[\w.-]+:)?)response>[\s\S]*?<\/\1response>/g;+const HREF_PATTERN = /<(?:[\w.-]+:)?href>\s*([^<]*?)\s*<\/(?:[\w.-]+:)?href>/;++export type NonCompliantCalDavProxy = {+ host: string;+ port: number;+ degradations: {+ blankLastModified: boolean;+ hideCollectionMembers: boolean;+ };+ stop: () => Promise<void>;+};++const decodeBody = (body: Buffer, contentEncoding: string): Buffer => {+ if (body.length === 0) {+ return body;+ }++ switch (contentEncoding) {+ case 'gzip':+ return gunzipSync(body);+ case 'deflate':+ return inflateSync(body);+ case 'br':+ return brotliDecompressSync(body);+ default:+ return body;+ }+};++const readBody = async (message: IncomingMessage): Promise<Buffer> => {+ const chunks: Buffer[] = [];++ for await (const chunk of message) {+ chunks.push(chunk as Buffer);+ }++ return Buffer.concat(chunks);+};++const blankEntityTags = (body: string): string =>+ body.replace(ENTITY_TAG_PATTERN, '<$1getetag />');++const blankLastModified = (body: string): string =>+ body.replace(LAST_MODIFIED_PATTERN, '<$1getlastmodified />');++const dropSyncCollectionReport = (body: string): string =>+ body.replace(SUPPORTED_REPORT_PATTERN, (report) =>+ report.includes('sync-collection') ? '' : report,+ );++const dropMemberResponses = (body: string): string =>+ body.replace(RESPONSE_PATTERN, (response) =>+ HREF_PATTERN.exec(response)?.[1].endsWith('/') ? response : '',+ );++export const startNonCompliantCalDavProxy = async ({+ targetHost,+ targetPort,+}: {+ targetHost: string;+ targetPort: number;+}): Promise<NonCompliantCalDavProxy> => {+ const degradations = {+ blankLastModified: false,+ hideCollectionMembers: false,+ };++ const server = createServer((incoming, outgoing) => {+ void readBody(incoming)+ .then((requestBody) => {+ const upstream = request(+ {+ host: targetHost,+ port: targetPort,+ method: incoming.method,+ path: incoming.url,+ headers: {+ ...incoming.headers,+ host: `${targetHost}:${targetPort}`,+ 'accept-encoding': 'identity',+ },+ },+ (upstreamResponse) => {+ void readBody(upstreamResponse)+ .then((responseBody) => {+ const headers = { ...upstreamResponse.headers };+ const contentEncoding = String(+ headers['content-encoding'] ?? '',+ );++ delete headers['content-length'];+ delete headers['transfer-encoding'];+ delete headers['content-encoding'];++ const decodedBody = decodeBody(responseBody, contentEncoding);+ const isXml = String(headers['content-type'] ?? '').includes(+ 'xml',+ );+ const rewrites = [blankEntityTags, dropSyncCollectionReport];++ if (degradations.blankLastModified) {+ rewrites.push(blankLastModified);+ }++ if (+ degradations.hideCollectionMembers &&+ incoming.method === 'PROPFIND'+ ) {+ rewrites.push(dropMemberResponses);+ }++ outgoing.writeHead(upstreamResponse.statusCode ?? 502, headers);+ outgoing.end(+ isXml+ ? rewrites.reduce(+ (body, rewrite) => rewrite(body),+ decodedBody.toString('utf8'),+ )+ : decodedBody,+ );+ })+ .catch(() => outgoing.destroy());+ },+ );++ upstream.on('error', () => outgoing.destroy());+ upstream.end(requestBody);+ })+ .catch(() => outgoing.destroy());+ });++ server.listen(0, PROXY_HOST);+ await once(server, 'listening');++ return {+ host: PROXY_HOST,+ port: (server.address() as AddressInfo).port,+ degradations,+ stop: async () => {+ server.closeAllConnections();+ server.close();+ await once(server, 'close');+ },+ };+};