The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Download tarball instead of yarn install when installing application (#20835)
martmullefba71a93864
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-package/application-package-fetcher.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-package/application-package-fetcher.service.tsindex 1e6fb73aa14..5032f780a03 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-package/application-package-fetcher.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-package/application-package-fetcher.service.ts@@ -1,12 +1,11 @@import { Injectable, Logger, type OnModuleInit } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';-import { execFile } from 'child_process';import { promises as fs } from 'fs';import { tmpdir } from 'os';import { join } from 'path';-import { promisify } from 'util';+import { isAxiosError } from 'axios';import { type Manifest } from 'twenty-shared/application';import { FileFolder } from 'twenty-shared/types';import { isDefined } from 'twenty-shared/utils';@@ -21,7 +20,6 @@ import {ApplicationExceptionCode,} from 'src/engine/core-modules/application/application.exception';import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';-import { YARN_ENGINE_DIRNAME } from 'src/engine/core-modules/application/application-package/constants/yarn-engine-dirname';import { assertValidNpmPackageName } from 'src/engine/core-modules/application/application-package/utils/assert-valid-npm-package-name.util';import { extractTarballSecurely } from 'src/engine/core-modules/application/application-package/utils/extract-tarball-securely.util';import { readJsonFileOrThrow } from 'src/engine/core-modules/application/application-package/utils/read-json-file.util';@@ -29,11 +27,10 @@ import { resolvePackageContentDir } from 'src/engine/core-modules/application/apimport { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils';+import { SecureHttpClientService } from 'src/engine/core-modules/secure-http-client/secure-http-client.service';import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';import { streamToBuffer } from 'src/utils/stream-to-buffer';-const execFilePromise = promisify(execFile);-const APP_FETCHER_TMPDIR = join(tmpdir(), 'twenty-app-fetcher');const RESOLUTION_TIMEOUT_MS = 30_000;@@ -51,6 +48,7 @@ export class ApplicationPackageFetcherService implements OnModuleInit {constructor(private readonly twentyConfigService: TwentyConfigService,private readonly fileStorageService: FileStorageService,+ private readonly secureHttpClientService: SecureHttpClientService,@InjectRepository(FileEntity)private readonly fileRepository: Repository<FileEntity>,@InjectRepository(ApplicationEntity)@@ -65,13 +63,6 @@ export class ApplicationPackageFetcherService implements OnModuleInit {}}- async resolveNpmPackage(- packageName: string,- targetVersion?: string,- ): Promise<ResolvedPackage> {- return this.resolveFromNpm(packageName, targetVersion);- }-async resolvePackage(appRegistration: ApplicationRegistrationEntity,options?: { targetVersion?: string },@@ -115,41 +106,53 @@ export class ApplicationPackageFetcherService implements OnModuleInit {try {const registryUrl = this.twentyConfigService.get('APP_REGISTRY_URL');-const authToken = this.twentyConfigService.get('APP_REGISTRY_TOKEN');assertValidNpmPackageName(packageName);const versionSpec = targetVersion ?? 'latest';- await this.writeNpmrc({- workDir,+ const tarballUrl = await this.fetchTarballUrl(+ registryUrl,packageName,+ versionSpec,+ authToken,+ );++ const tarballBuffer = await this.downloadTarball(+ tarballUrl,registryUrl,authToken,- });- await this.setupYarnEngine(workDir);- await this.writeMinimalPackageJson(workDir, packageName, versionSpec);- await this.runYarnInstall(workDir);+ );+ const tarballPath = join(workDir, 'package.tgz');- const packageDir = join(workDir, 'node_modules', packageName);+ await fs.writeFile(tarballPath, tarballBuffer);+ await extractTarballSecurely(tarballPath, workDir);+ await fs.rm(tarballPath);++ const contentDir = await resolvePackageContentDir(workDir);const manifest = await readJsonFileOrThrow<Manifest>(- packageDir,+ contentDir,'manifest.json',);const packageJson = await readJsonFileOrThrow<PackageJson>(- packageDir,+ contentDir,'package.json',);return {- extractedDir: packageDir,+ extractedDir: contentDir,cleanupDir: workDir,manifest,packageJson,};} catch (error) {await this.cleanupExtractedDir(workDir);++ if (error instanceof ApplicationException) {+ throw error;+ }+throw new ApplicationException(`Failed to resolve npm package ${packageName}: ${error}`,ApplicationExceptionCode.PACKAGE_RESOLUTION_FAILED,@@ -224,105 +227,83 @@ export class ApplicationPackageFetcherService implements OnModuleInit {}}- // Note: .npmrc settings take precedence over publishConfig.registry in- // individual packages. This is correct for our use case since we want- // to control the registry at the resolver level.- private async writeNpmrc(config: {- workDir: string;- packageName: string;- registryUrl: string;- authToken?: string;- }): Promise<void> {- const lines: string[] = [];- const registryHost = new URL(config.registryUrl).host;-- if (config.packageName.startsWith('@')) {- const scope = config.packageName.split('/')[0];-- lines.push(`${scope}:registry=${config.registryUrl}`);- } else if (config.registryUrl !== 'https://registry.npmjs.org') {- lines.push(`registry=${config.registryUrl}`);- }-- if (config.authToken) {- lines.push(`//${registryHost}/:_authToken=${config.authToken}`);- }-- if (lines.length > 0) {- await fs.writeFile(- join(config.workDir, '.npmrc'),- lines.join('\n') + '\n',- );- }- }-- private async setupYarnEngine(workDir: string): Promise<void> {- await fs.cp(YARN_ENGINE_DIRNAME, workDir, { recursive: true });- }-- private async writeMinimalPackageJson(- workDir: string,+ private async fetchTarballUrl(+ registryUrl: string,packageName: string,versionSpec: string,- ): Promise<void> {- const packageJson = {- name: 'twenty-app-resolver-workspace',- private: true,- dependencies: {- [packageName]: versionSpec,- },+ authToken?: string,+ ): Promise<string> {+ const encodedName = encodeURIComponent(packageName);+ const baseUrl = registryUrl.replace(/\/$/, '');+ const metadataUrl = `${baseUrl}/${encodedName}/${versionSpec}`;++ const headers: Record<string, string> = {+ Accept: 'application/json',};- await fs.writeFile(- join(workDir, 'package.json'),- JSON.stringify(packageJson, null, 2),- );- }+ const httpClient = this.secureHttpClientService.getHttpClient({+ timeout: RESOLUTION_TIMEOUT_MS,+ ...(isDefined(authToken)+ ? {+ headers: {+ Authorization: `Bearer ${authToken}`,+ },+ }+ : {}),+ });- private async resolveLocalYarnPath(workDir: string): Promise<string> {- const yarnrcPath = join(workDir, '.yarnrc.yml');- const yarnrcContent = await fs.readFile(yarnrcPath, 'utf-8');- const match = yarnrcContent.match(/^yarnPath:\s*(.+)$/m);+ let response;- if (!match) {+ try {+ response = await httpClient.get<{+ dist?: { tarball?: string };+ }>(metadataUrl, { headers });+ } catch (error) {throw new ApplicationException(- 'yarnPath not found in .yarnrc.yml',+ `Registry returned ${isAxiosError(error) ? error.response?.status : 'unknown error'} for ${packageName}@${versionSpec}`,ApplicationExceptionCode.PACKAGE_RESOLUTION_FAILED,);}- return join(workDir, match[1].trim());+ const tarballUrl = response.data?.dist?.tarball;++ if (!tarballUrl) {+ throw new ApplicationException(+ `No tarball URL in registry metadata for ${packageName}@${versionSpec}`,+ ApplicationExceptionCode.PACKAGE_RESOLUTION_FAILED,+ );+ }++ return tarballUrl;}- private async runYarnInstall(workDir: string): Promise<void> {- const localYarnPath = await this.resolveLocalYarnPath(workDir);+ private async downloadTarball(+ tarballUrl: string,+ registryUrl: string,+ authToken?: string,+ ): Promise<Buffer> {+ const headers: Record<string, string> = {};- const { NODE_OPTIONS: _nodeOptions, ...cleanEnv } = process.env;+ const isSameHost = new URL(tarballUrl).host === new URL(registryUrl).host;++ if (authToken && isSameHost) {+ headers['Authorization'] = `Bearer ${authToken}`;+ }++ const httpClient = this.secureHttpClientService.getHttpClient({+ timeout: RESOLUTION_TIMEOUT_MS,+ });try {- await execFilePromise(- process.execPath,- [localYarnPath, 'install', '--no-immutable'],- {- cwd: workDir,- env: cleanEnv,- timeout: RESOLUTION_TIMEOUT_MS,- },- );- } catch (error: unknown) {- const stderr =- isDefined(error) &&- typeof error === 'object' &&- 'stderr' in error &&- typeof (error as { stderr: unknown }).stderr === 'string'- ? (error as { stderr: string }).stderr- : undefined;-- const message =- stderr ?? (error instanceof Error ? error.message : String(error));+ const response = await httpClient.get(tarballUrl, {+ headers,+ responseType: 'arraybuffer',+ });+ return Buffer.from(response.data);+ } catch (error) {throw new ApplicationException(- `yarn install failed: ${message}`,+ `Failed to download tarball: ${isAxiosError(error) ? error.response?.status : error}`,ApplicationExceptionCode.PACKAGE_RESOLUTION_FAILED,);}diff --git a/packages/twenty-server/src/engine/core-modules/application/application-package/application-package.module.ts b/packages/twenty-server/src/engine/core-modules/application/application-package/application-package.module.tsindex 04b7a085d68..646de25ab66 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-package/application-package.module.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-package/application-package.module.ts@@ -6,12 +6,14 @@ import { ApplicationVersionValidationService } from 'src/engine/core-modules/appimport { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';import { FileStorageModule } from 'src/engine/core-modules/file-storage/file-storage.module';import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';+import { SecureHttpClientModule } from 'src/engine/core-modules/secure-http-client/secure-http-client.module';import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';import { UpgradeModule } from 'src/engine/core-modules/upgrade/upgrade.module';@Module({imports: [FileStorageModule,+ SecureHttpClientModule,TwentyConfigModule,UpgradeModule,TypeOrmModule.forFeature([FileEntity, ApplicationEntity]),