Skip to content

The open alternative to Salesforce, designed for AI.

Download tarball instead of yarn install when installing application (#20835)

martmullefba71a93864
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-package/application-package-fetcher.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-package/application-package-fetcher.service.ts
index 1e6fb73aa14..5032f780a03 100644
--- a/packages/twenty-server/src/engine/core-modules/application/application-package/application-package-fetcher.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/application/application-package/application-package-fetcher.service.ts
@@ -1,12 +1,11 @@
import { Injectable, Logger, type OnModuleInit } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
-import { execFile } from 'child_process';
import { promises as fs } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
-import { promisify } from 'util';
+import { isAxiosError } from 'axios';
import { type Manifest } from 'twenty-shared/application';
import { FileFolder } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
@@ -21,7 +20,6 @@ import {
ApplicationExceptionCode,
} from 'src/engine/core-modules/application/application.exception';
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
-import { YARN_ENGINE_DIRNAME } from 'src/engine/core-modules/application/application-package/constants/yarn-engine-dirname';
import { assertValidNpmPackageName } from 'src/engine/core-modules/application/application-package/utils/assert-valid-npm-package-name.util';
import { extractTarballSecurely } from 'src/engine/core-modules/application/application-package/utils/extract-tarball-securely.util';
import { readJsonFileOrThrow } from 'src/engine/core-modules/application/application-package/utils/read-json-file.util';
@@ -29,11 +27,10 @@ import { resolvePackageContentDir } from 'src/engine/core-modules/application/ap
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils';
+import { SecureHttpClientService } from 'src/engine/core-modules/secure-http-client/secure-http-client.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
-const execFilePromise = promisify(execFile);
-
const APP_FETCHER_TMPDIR = join(tmpdir(), 'twenty-app-fetcher');
const RESOLUTION_TIMEOUT_MS = 30_000;
@@ -51,6 +48,7 @@ export class ApplicationPackageFetcherService implements OnModuleInit {
constructor(
private readonly twentyConfigService: TwentyConfigService,
private readonly fileStorageService: FileStorageService,
+ private readonly secureHttpClientService: SecureHttpClientService,
@InjectRepository(FileEntity)
private readonly fileRepository: Repository<FileEntity>,
@InjectRepository(ApplicationEntity)
@@ -65,13 +63,6 @@ export class ApplicationPackageFetcherService implements OnModuleInit {
}
}
- async resolveNpmPackage(
- packageName: string,
- targetVersion?: string,
- ): Promise<ResolvedPackage> {
- return this.resolveFromNpm(packageName, targetVersion);
- }
-
async resolvePackage(
appRegistration: ApplicationRegistrationEntity,
options?: { targetVersion?: string },
@@ -115,41 +106,53 @@ export class ApplicationPackageFetcherService implements OnModuleInit {
try {
const registryUrl = this.twentyConfigService.get('APP_REGISTRY_URL');
-
const authToken = this.twentyConfigService.get('APP_REGISTRY_TOKEN');
assertValidNpmPackageName(packageName);
const versionSpec = targetVersion ?? 'latest';
- await this.writeNpmrc({
- workDir,
+ const tarballUrl = await this.fetchTarballUrl(
+ registryUrl,
packageName,
+ versionSpec,
+ authToken,
+ );
+
+ const tarballBuffer = await this.downloadTarball(
+ tarballUrl,
registryUrl,
authToken,
- });
- await this.setupYarnEngine(workDir);
- await this.writeMinimalPackageJson(workDir, packageName, versionSpec);
- await this.runYarnInstall(workDir);
+ );
+ const tarballPath = join(workDir, 'package.tgz');
- const packageDir = join(workDir, 'node_modules', packageName);
+ await fs.writeFile(tarballPath, tarballBuffer);
+ await extractTarballSecurely(tarballPath, workDir);
+ await fs.rm(tarballPath);
+
+ const contentDir = await resolvePackageContentDir(workDir);
const manifest = await readJsonFileOrThrow<Manifest>(
- packageDir,
+ contentDir,
'manifest.json',
);
const packageJson = await readJsonFileOrThrow<PackageJson>(
- packageDir,
+ contentDir,
'package.json',
);
return {
- extractedDir: packageDir,
+ extractedDir: contentDir,
cleanupDir: workDir,
manifest,
packageJson,
};
} catch (error) {
await this.cleanupExtractedDir(workDir);
+
+ if (error instanceof ApplicationException) {
+ throw error;
+ }
+
throw new ApplicationException(
`Failed to resolve npm package ${packageName}: ${error}`,
ApplicationExceptionCode.PACKAGE_RESOLUTION_FAILED,
@@ -224,105 +227,83 @@ export class ApplicationPackageFetcherService implements OnModuleInit {
}
}
- // Note: .npmrc settings take precedence over publishConfig.registry in
- // individual packages. This is correct for our use case since we want
- // to control the registry at the resolver level.
- private async writeNpmrc(config: {
- workDir: string;
- packageName: string;
- registryUrl: string;
- authToken?: string;
- }): Promise<void> {
- const lines: string[] = [];
- const registryHost = new URL(config.registryUrl).host;
-
- if (config.packageName.startsWith('@')) {
- const scope = config.packageName.split('/')[0];
-
- lines.push(`${scope}:registry=${config.registryUrl}`);
- } else if (config.registryUrl !== 'https://registry.npmjs.org') {
- lines.push(`registry=${config.registryUrl}`);
- }
-
- if (config.authToken) {
- lines.push(`//${registryHost}/:_authToken=${config.authToken}`);
- }
-
- if (lines.length > 0) {
- await fs.writeFile(
- join(config.workDir, '.npmrc'),
- lines.join('\n') + '\n',
- );
- }
- }
-
- private async setupYarnEngine(workDir: string): Promise<void> {
- await fs.cp(YARN_ENGINE_DIRNAME, workDir, { recursive: true });
- }
-
- private async writeMinimalPackageJson(
- workDir: string,
+ private async fetchTarballUrl(
+ registryUrl: string,
packageName: string,
versionSpec: string,
- ): Promise<void> {
- const packageJson = {
- name: 'twenty-app-resolver-workspace',
- private: true,
- dependencies: {
- [packageName]: versionSpec,
- },
+ authToken?: string,
+ ): Promise<string> {
+ const encodedName = encodeURIComponent(packageName);
+ const baseUrl = registryUrl.replace(/\/$/, '');
+ const metadataUrl = `${baseUrl}/${encodedName}/${versionSpec}`;
+
+ const headers: Record<string, string> = {
+ Accept: 'application/json',
};
- await fs.writeFile(
- join(workDir, 'package.json'),
- JSON.stringify(packageJson, null, 2),
- );
- }
+ const httpClient = this.secureHttpClientService.getHttpClient({
+ timeout: RESOLUTION_TIMEOUT_MS,
+ ...(isDefined(authToken)
+ ? {
+ headers: {
+ Authorization: `Bearer ${authToken}`,
+ },
+ }
+ : {}),
+ });
- private async resolveLocalYarnPath(workDir: string): Promise<string> {
- const yarnrcPath = join(workDir, '.yarnrc.yml');
- const yarnrcContent = await fs.readFile(yarnrcPath, 'utf-8');
- const match = yarnrcContent.match(/^yarnPath:\s*(.+)$/m);
+ let response;
- if (!match) {
+ try {
+ response = await httpClient.get<{
+ dist?: { tarball?: string };
+ }>(metadataUrl, { headers });
+ } catch (error) {
throw new ApplicationException(
- 'yarnPath not found in .yarnrc.yml',
+ `Registry returned ${isAxiosError(error) ? error.response?.status : 'unknown error'} for ${packageName}@${versionSpec}`,
ApplicationExceptionCode.PACKAGE_RESOLUTION_FAILED,
);
}
- return join(workDir, match[1].trim());
+ const tarballUrl = response.data?.dist?.tarball;
+
+ if (!tarballUrl) {
+ throw new ApplicationException(
+ `No tarball URL in registry metadata for ${packageName}@${versionSpec}`,
+ ApplicationExceptionCode.PACKAGE_RESOLUTION_FAILED,
+ );
+ }
+
+ return tarballUrl;
}
- private async runYarnInstall(workDir: string): Promise<void> {
- const localYarnPath = await this.resolveLocalYarnPath(workDir);
+ private async downloadTarball(
+ tarballUrl: string,
+ registryUrl: string,
+ authToken?: string,
+ ): Promise<Buffer> {
+ const headers: Record<string, string> = {};
- const { NODE_OPTIONS: _nodeOptions, ...cleanEnv } = process.env;
+ const isSameHost = new URL(tarballUrl).host === new URL(registryUrl).host;
+
+ if (authToken && isSameHost) {
+ headers['Authorization'] = `Bearer ${authToken}`;
+ }
+
+ const httpClient = this.secureHttpClientService.getHttpClient({
+ timeout: RESOLUTION_TIMEOUT_MS,
+ });
try {
- await execFilePromise(
- process.execPath,
- [localYarnPath, 'install', '--no-immutable'],
- {
- cwd: workDir,
- env: cleanEnv,
- timeout: RESOLUTION_TIMEOUT_MS,
- },
- );
- } catch (error: unknown) {
- const stderr =
- isDefined(error) &&
- typeof error === 'object' &&
- 'stderr' in error &&
- typeof (error as { stderr: unknown }).stderr === 'string'
- ? (error as { stderr: string }).stderr
- : undefined;
-
- const message =
- stderr ?? (error instanceof Error ? error.message : String(error));
+ const response = await httpClient.get(tarballUrl, {
+ headers,
+ responseType: 'arraybuffer',
+ });
+ return Buffer.from(response.data);
+ } catch (error) {
throw new ApplicationException(
- `yarn install failed: ${message}`,
+ `Failed to download tarball: ${isAxiosError(error) ? error.response?.status : error}`,
ApplicationExceptionCode.PACKAGE_RESOLUTION_FAILED,
);
}
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-package/application-package.module.ts b/packages/twenty-server/src/engine/core-modules/application/application-package/application-package.module.ts
index 04b7a085d68..646de25ab66 100644
--- a/packages/twenty-server/src/engine/core-modules/application/application-package/application-package.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/application/application-package/application-package.module.ts
@@ -6,12 +6,14 @@ import { ApplicationVersionValidationService } from 'src/engine/core-modules/app
import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
import { FileStorageModule } from 'src/engine/core-modules/file-storage/file-storage.module';
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
+import { SecureHttpClientModule } from 'src/engine/core-modules/secure-http-client/secure-http-client.module';
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
import { UpgradeModule } from 'src/engine/core-modules/upgrade/upgrade.module';
@Module({
imports: [
FileStorageModule,
+ SecureHttpClientModule,
TwentyConfigModule,
UpgradeModule,
TypeOrmModule.forFeature([FileEntity, ApplicationEntity]),