The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Fetch the latest S3 frontend HTML for each document request (#26835)
Félix Malfait34db6464d416
diff --git a/packages/twenty-server/src/engine/core-modules/frontend/__tests__/frontend.service.spec.ts b/packages/twenty-server/src/engine/core-modules/frontend/__tests__/frontend.service.spec.tsindex 517a03d0be8..242327217c8 100644--- a/packages/twenty-server/src/engine/core-modules/frontend/__tests__/frontend.service.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/frontend/__tests__/frontend.service.spec.ts@@ -12,6 +12,7 @@ import request from 'supertest';import { type ClientConfigService } from 'src/engine/core-modules/client-config/services/client-config.service';import { type WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service';import { FrontendService } from 'src/engine/core-modules/frontend/frontend.service';+import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';jest.mock(@@ -23,6 +24,13 @@ jest.mock(() => ({ WorkspaceDomainsService: class {} }),);+jest.mock(+ 'src/engine/core-modules/twenty-config/twenty-config.service',+ () => ({+ TwentyConfigService: class {},+ }),+);+jest.mock('src/engine/core-modules/client-config/client-config.module', () => ({ClientConfigModule: class {},}));@@ -51,6 +59,26 @@ describe('frontend HTML delivery', () => {let directory: string;let app: INestApplication;+ const createApplication = async (indexUrl?: string) => {+ const service = new FrontendService(+ { getClientConfig } as unknown as ClientConfigService,+ { resolveWorkspaceAndPublicDomain } as unknown as WorkspaceDomainsService,+ directory,+ { get: () => indexUrl } as unknown as TwentyConfigService,+ );+ const module = await Test.createTestingModule({+ imports: [FrontendModule],+ controllers: [HealthController],+ })+ .overrideProvider(FrontendService)+ .useValue(service)+ .compile();++ app = module.createNestApplication();+ app.getHttpAdapter().getInstance().set('trust proxy', 'loopback');+ await app.init();+ };+beforeEach(async () => {jest.useRealTimers();getClientConfig.mockReset().mockResolvedValue(config);@@ -70,45 +98,65 @@ describe('frontend HTML delivery', () => {join(directory, 'index.html'),'<html><head><!-- BEGIN: Twenty Config --><script>window._env_={REACT_APP_SERVER_BASE_URL:"wrong"}</script><!-- END: Twenty Config --></head><body><div id="root"></div></body></html>',);- const service = new FrontendService(- { getClientConfig } as unknown as ClientConfigService,- { resolveWorkspaceAndPublicDomain } as unknown as WorkspaceDomainsService,- directory,- );- const module = await Test.createTestingModule({- imports: [FrontendModule],- controllers: [HealthController],- })- .overrideProvider(FrontendService)- .useValue(service)- .compile();-- app = module.createNestApplication();- app.getHttpAdapter().getInstance().set('trust proxy', 'loopback');- await app.init();+ await createApplication();});afterEach(async () => {await app.close();rmSync(directory, { recursive: true, force: true });+ jest.restoreAllMocks();+ jest.useRealTimers();});- it('initializes CLI application contexts with bundled HTML and no HTTP adapter', async () => {- const module = await Test.createTestingModule({ imports: [FrontendModule] })- .overrideProvider(FrontendService)- .useValue(- new FrontendService(- { getClientConfig } as unknown as ClientConfigService,- {- resolveWorkspaceAndPublicDomain,- } as unknown as WorkspaceDomainsService,- directory,- ),- )- .compile();- expect(module.get(HttpAdapterHost).httpAdapter).toBeUndefined();- await expect(module.init()).resolves.toBeDefined();- await module.close();+ it.each([undefined, 'https://frontend.s3.example.com/index.html'])(+ 'initializes CLI application contexts without fetching HTML (origin: %s)',+ async (indexUrl) => {+ const fetchTemplate = jest.spyOn(globalThis, 'fetch');+ const module = await Test.createTestingModule({+ imports: [FrontendModule],+ })+ .overrideProvider(FrontendService)+ .useValue(+ new FrontendService(+ { getClientConfig } as unknown as ClientConfigService,+ {+ resolveWorkspaceAndPublicDomain,+ } as unknown as WorkspaceDomainsService,+ directory,+ { get: () => indexUrl } as unknown as TwentyConfigService,+ ),+ )+ .compile();+ expect(module.get(HttpAdapterHost).httpAdapter).toBeUndefined();+ await expect(module.init()).resolves.toBeDefined();+ expect(fetchTemplate).not.toHaveBeenCalled();+ await module.close();+ },+ );++ it('serves bundled HTML without an external request when the URL is unset', async () => {+ const fetchTemplate = jest.spyOn(globalThis, 'fetch');+ const response = await request(app.getHttpServer())+ .get('/')+ .set('Accept', 'text/html')+ .expect(200);+ expect(response.text).toContain('<div id="root"></div>');+ expect(response.text).toContain('twenty-client-config');+ expect(fetchTemplate).not.toHaveBeenCalled();+ });++ it('preserves API-only development when neither bundled HTML nor a URL exists', async () => {+ await app.close();+ rmSync(join(directory, 'index.html'));+ const fetchTemplate = jest.spyOn(globalThis, 'fetch');+ await createApplication();+ await request(app.getHttpServer()).get('/healthz').expect(200);+ await request(app.getHttpServer())+ .get('/')+ .set('Accept', 'text/html')+ .expect(404);+ expect(fetchTemplate).not.toHaveBeenCalled();+ expect(getClientConfig).not.toHaveBeenCalled();});it.each(['/', '/index.html', '/objects/people'])(@@ -353,4 +401,139 @@ describe('frontend HTML delivery', () => {);expect(response.text).toBeUndefined();});++ describe('remote HTML template', () => {+ const indexUrl = 'https://frontend.s3.example.com/index.html';+ const remoteTemplate =+ '<html><head></head><body>Remote release</body></html>';+ let fetchTemplate: jest.SpiedFunction<typeof fetch>;++ beforeEach(async () => {+ await app.close();+ rmSync(join(directory, 'index.html'));+ fetchTemplate = jest+ .spyOn(globalThis, 'fetch')+ .mockImplementation(async () => new Response(remoteTemplate));+ await createApplication(indexUrl);+ });++ it('fetches the current HTML per document and applies each workspace policy', async () => {+ expect(fetchTemplate).not.toHaveBeenCalled();+ for (const host of ['customer.twenty.test', 'other.twenty.test']) {+ fetchTemplate.mockResolvedValueOnce(+ new Response(`<head></head><body>Latest ${host}</body>`),+ );+ const response = await request(app.getHttpServer())+ .get('/')+ .set('Host', host)+ .set('X-Forwarded-Proto', 'https')+ .set('Accept', 'text/html')+ .expect(200);+ expect(response.text).toContain(`Latest ${host}`);+ expect(response.text).toContain('twenty-client-config');+ expect(response.headers['cache-control']).toBe('no-store');+ expect(response.headers['content-security-policy']).toBe(+ host === 'customer.twenty.test'+ ? "frame-ancestors 'self' https://portal.customer.com; object-src 'none'; base-uri 'self'"+ : "frame-ancestors 'self'; object-src 'none'; base-uri 'self'",+ );+ }+ expect(fetchTemplate).toHaveBeenCalledTimes(2);+ expect(fetchTemplate).toHaveBeenCalledWith(indexUrl, {+ signal: expect.any(AbortSignal),+ redirect: 'error',+ });+ });++ it('rejects isolated origins without contacting an unavailable HTML origin', async () => {+ resolveWorkspaceAndPublicDomain.mockResolvedValue({+ workspace: undefined,+ isIsolatedOrigin: true,+ });+ fetchTemplate.mockRejectedValue(new Error('Origin unavailable'));+ await request(app.getHttpServer())+ .get('/')+ .set('Accept', 'text/html')+ .expect(404);+ expect(fetchTemplate).not.toHaveBeenCalled();+ expect(getClientConfig).not.toHaveBeenCalled();+ });++ it.each([undefined, '1'])(+ 'stops oversized HTML downloads independently of Content-Length: %s',+ async (contentLength) => {+ const cancel = jest.fn();+ const body = new ReadableStream<Uint8Array>({+ pull(controller) {+ controller.enqueue(+ Buffer.from('<head></head>' + 'x'.repeat(64 * 1024)),+ );+ },+ cancel,+ });+ fetchTemplate.mockResolvedValueOnce(+ new Response(body, {+ headers: contentLength ? { 'Content-Length': contentLength } : {},+ }),+ );+ const response = await request(app.getHttpServer())+ .get('/')+ .set('Accept', 'text/html')+ .expect(503);+ expect(response.headers['cache-control']).toBe('no-store');+ expect(response.headers['x-frame-options']).toBe('SAMEORIGIN');+ expect(cancel).toHaveBeenCalled();+ },+ );++ it.each(['network', 'http', 'invalid HTML'])(+ 'returns 503 on %s failure and recovers on the next request',+ async (failure) => {+ if (failure === 'network') {+ fetchTemplate.mockRejectedValueOnce(new Error('Origin unavailable'));+ } else {+ fetchTemplate.mockResolvedValueOnce(+ new Response('Invalid template', {+ status: failure === 'http' ? 503 : 200,+ }),+ );+ }+ const response = await request(app.getHttpServer())+ .get('/')+ .set('Accept', 'text/html')+ .expect(503);+ expect(response.headers['cache-control']).toBe('no-store');+ expect(response.headers['x-frame-options']).toBe('SAMEORIGIN');+ const recovered = await request(app.getHttpServer())+ .get('/')+ .set('Accept', 'text/html')+ .expect(200);+ expect(recovered.text).toContain('Remote release');+ },+ );++ it('bounds the wait for an unresponsive origin', async () => {+ fetchTemplate.mockImplementationOnce(+ async (_url, options) =>+ new Promise((_resolve, reject) => {+ options?.signal?.addEventListener('abort', () =>+ reject(options.signal?.reason),+ );+ }),+ );+ await request(app.getHttpServer())+ .get('/')+ .set('Accept', 'text/html')+ .expect(503);+ }, 10_000);++ it('does not fetch HTML for health checks or missing API routes', async () => {+ await request(app.getHttpServer()).get('/healthz').expect(200);+ await request(app.getHttpServer())+ .get('/graphql/missing')+ .set('Accept', 'text/html')+ .expect(404);+ expect(fetchTemplate).not.toHaveBeenCalled();+ });+ });});diff --git a/packages/twenty-server/src/engine/core-modules/frontend/frontend.service.ts b/packages/twenty-server/src/engine/core-modules/frontend/frontend.service.tsindex 3d9f1fd23a8..d649c74eeb6 100644--- a/packages/twenty-server/src/engine/core-modules/frontend/frontend.service.ts+++ b/packages/twenty-server/src/engine/core-modules/frontend/frontend.service.ts@@ -2,6 +2,8 @@ import { Inject, Injectable, Logger } from '@nestjs/common';import { existsSync, readFileSync } from 'fs';import { join } from 'path';+import { Readable } from 'stream';+import { type ReadableStream } from 'stream/web';import { type NextFunction, type Request, type Response } from 'express';import { isDefined, normalizeAllowedIframeOrigin } from 'twenty-shared/utils';@@ -10,32 +12,83 @@ import { ClientConfigService } from 'src/engine/core-modules/client-config/serviimport { WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service';import { isFrontendDocumentRequest } from 'src/engine/core-modules/frontend/utils/is-frontend-document-request.util';import { renderFrontendHtml } from 'src/engine/core-modules/frontend/utils/render-frontend-html.util';+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception';import { getRequestBaseUrl } from 'src/utils/get-request-base-url.util';+import { streamToBuffer } from 'src/utils/stream-to-buffer';++const MAX_FRONTEND_HTML_SIZE_BYTES = 1024 * 1024;@Injectable()export class FrontendService {private readonly logger = new Logger(FrontendService.name);private readonly template: string | undefined;+ private readonly indexUrl: string | undefined;constructor(private readonly clientConfigService: ClientConfigService,private readonly workspaceDomainsService: WorkspaceDomainsService,@Inject('FRONTEND_PATH') readonly frontPath: string,+ twentyConfigService: TwentyConfigService,) {+ this.indexUrl = twentyConfigService.get('FRONTEND_INDEX_URL');++ if (isDefined(this.indexUrl)) {+ return;+ }+const indexPath = join(this.frontPath, 'index.html');if (existsSync(indexPath)) {this.template = readFileSync(indexPath, 'utf8');- if (!this.template.includes('</head>')) {- throw new Error('Frontend index.html must contain a closing head tag');- }+ this.validateTemplate(this.template);}}get isEnabled(): boolean {- return isDefined(this.template);+ return isDefined(this.template) || isDefined(this.indexUrl);+ }++ private validateTemplate(template: string): void {+ if (!template.includes('</head>')) {+ throw new Error('Frontend index.html must contain a closing head tag');+ }+ }++ private async getTemplate(): Promise<string> {+ if (!isDefined(this.indexUrl)) {+ if (!isDefined(this.template)) {+ throw new Error('Frontend HTML is not configured');+ }++ return this.template;+ }++ const response = await fetch(this.indexUrl, {+ signal: AbortSignal.timeout(5_000),+ redirect: 'error',+ });++ if (!response.ok) {+ await response.body?.cancel();+ throw new Error(`Unable to fetch frontend HTML: HTTP ${response.status}`);+ }++ if (!isDefined(response.body)) {+ throw new Error('Frontend index.html must have a response body');+ }++ const template = (+ await streamToBuffer(+ Readable.fromWeb(response.body as ReadableStream<Uint8Array>),+ MAX_FRONTEND_HTML_SIZE_BYTES,+ )+ ).toString('utf8');++ this.validateTemplate(template);++ return template;}async serveDocument(@@ -43,7 +96,7 @@ export class FrontendService {response: Response,next: NextFunction,): Promise<void> {- if (!isDefined(this.template) || !isFrontendDocumentRequest(request)) {+ if (!this.isEnabled || !isFrontendDocumentRequest(request)) {next();return;@@ -60,20 +113,15 @@ export class FrontendService {);try {- const [clientConfig, { workspace, isIsolatedOrigin }] = await Promise.all(- [- this.clientConfigService.getClientConfig(),- this.workspaceDomainsService- .resolveWorkspaceAndPublicDomain(getRequestBaseUrl(request))- .catch((error: unknown) => {- if (error === WorkspaceNotFoundDefaultError) {- return { workspace: undefined, isIsolatedOrigin: false };- }-- throw error;- }),- ],- );+ const { workspace, isIsolatedOrigin } = await this.workspaceDomainsService+ .resolveWorkspaceAndPublicDomain(getRequestBaseUrl(request))+ .catch((error: unknown) => {+ if (error === WorkspaceNotFoundDefaultError) {+ return { workspace: undefined, isIsolatedOrigin: false };+ }++ throw error;+ });if (isIsolatedOrigin) {response.status(404).end();@@ -81,6 +129,11 @@ export class FrontendService {return;}+ const [template, clientConfig] = await Promise.all([+ this.getTemplate(),+ this.clientConfigService.getClientConfig(),+ ]);+const allowedOrigins = (workspace?.allowedIframeOrigins ?? []).map(normalizeAllowedIframeOrigin).filter(isDefined);@@ -93,9 +146,7 @@ export class FrontendService {response.removeHeader('X-Frame-Options');}- response- .type('html')- .end(renderFrontendHtml(this.template, clientConfig));+ response.type('html').end(renderFrontendHtml(template, clientConfig));} catch (error) {this.logger.error('Unable to serve frontend document', error);responsediff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.tsindex bf2875bae87..87317d096ed 100644--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts@@ -1147,6 +1147,22 @@ export class ConfigVariables {@IsOptional()FRONTEND_URL: string;+ @ConfigVariablesMetadata({+ group: ConfigVariablesGroup.SERVER_CONFIG,+ description:+ 'Direct HTTPS origin URL for deployments that publish frontend HTML separately. Leave unset for standard self-hosted installations to use bundled HTML.',+ type: ConfigVariableType.STRING,+ isEnvOnly: true,+ isHiddenInAdminPanel: true,+ })+ @IsUrl({+ protocols: ['https'],+ require_tld: false,+ require_protocol: true,+ })+ @IsOptional()+ FRONTEND_INDEX_URL: string | undefined;+@ConfigVariablesMetadata({group: ConfigVariablesGroup.SERVER_CONFIG,description:diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.tsindex 88d92e1413a..ebe3e2aa710 100644--- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts@@ -1,5 +1,6 @@import { ConfigService } from '@nestjs/config';import { Test, type TestingModule } from '@nestjs/testing';+import { validateSync } from 'class-validator';import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants';@@ -38,6 +39,25 @@ describe('EnvironmentConfigDriver', () => {expect(driver).toBeDefined();});+ it.each([+ [undefined, true],+ ['https://frontend.s3.example.com/index.html', true],+ ['http://frontend.s3.example.com/index.html', false],+ ['ftp://frontend.s3.example.com/index.html', false],+ ['frontend.s3.example.com/index.html', false],+ ])('validates the optional frontend origin %s', (indexUrl, isValid) => {+ const config = new ConfigVariables();++ expect(config.FRONTEND_INDEX_URL).toBeUndefined();+ config.FRONTEND_INDEX_URL = indexUrl;++ const errors = validateSync(config, { strictGroups: true }).filter(+ (error) => error.property === 'FRONTEND_INDEX_URL',+ );++ expect(errors.length === 0).toBe(isValid);+ });+describe('get', () => {it('should return value from config service when available', () => {const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;