The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Log the app and workspace when an application hits the API rate limit (#27655)
martmull1d3000ce0025
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.tsindex 4d4263913c4..32f75fec59d 100644--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts@@ -1,4 +1,4 @@-import { Inject, Injectable } from '@nestjs/common';+import { Inject, Injectable, Logger } from '@nestjs/common';import { QUERY_MAX_RECORDS_FROM_RELATION } from 'twenty-shared/constants';import { FieldMetadataType, type ObjectRecord } from 'twenty-shared/types';@@ -20,6 +20,7 @@ import {CommonQueryRunnerExceptionCode,} from 'src/engine/api/common/common-query-runners/errors/common-query-runner.exception';import { STANDARD_ERROR_MESSAGE } from 'src/engine/api/common/common-query-runners/errors/standard-error-message.constant';+import { buildApplicationApiRateLimitedLogLine } from 'src/engine/api/common/common-query-runners/utils/build-application-api-rate-limited-log-line.util';import { buildMutationQueryBuilder } from 'src/engine/api/common/common-query-runners/utils/build-mutation-query-builder.util';import { computeMaxFieldCountPerRecord } from 'src/engine/api/common/common-query-runners/utils/compute-max-field-count-per-record.util';import { isRecordFilterEmpty } from 'src/engine/api/common/common-query-runners/utils/is-record-filter-empty.util';@@ -73,6 +74,8 @@ export abstract class CommonBaseQueryRunnerService<Args extends CommonQueryArgs,Output extends CommonQueryResult,> {+ private readonly logger = new Logger(CommonBaseQueryRunnerService.name);+@Inject()protected readonly workspaceQueryHookService: WorkspaceQueryHookService;@Inject()@@ -547,7 +550,7 @@ export abstract class CommonBaseQueryRunnerService<isDefined(error.exhaustedScope) &&error.exhaustedScope.isDefault) {- await this.incrementApiSpeedCeilingMetrics({+ await this.reportApiSpeedCeilingHit({authContext,exhaustedScope: error.exhaustedScope,});@@ -557,7 +560,7 @@ export abstract class CommonBaseQueryRunnerService<}}- private async incrementApiSpeedCeilingMetrics({+ private async reportApiSpeedCeilingHit({authContext,exhaustedScope,}: {@@ -584,6 +587,10 @@ export abstract class CommonBaseQueryRunnerService<source_type: authContext.application.sourceType,},});++ this.logger.warn(+ buildApplicationApiRateLimitedLogLine({ authContext, exhaustedScope }),+ );}}diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/utils/__tests__/build-application-api-rate-limited-log-line.util.spec.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/utils/__tests__/build-application-api-rate-limited-log-line.util.spec.tsnew file mode 100644index 00000000000..5c86674416a--- /dev/null+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/utils/__tests__/build-application-api-rate-limited-log-line.util.spec.ts@@ -0,0 +1,61 @@+import { buildApplicationApiRateLimitedLogLine } from 'src/engine/api/common/common-query-runners/utils/build-application-api-rate-limited-log-line.util';+import { type ApplicationWorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';+import { type ExhaustedScope } from 'src/engine/core-modules/usage-limit/types/exhausted-scope.type';+import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';++const authContext = {+ type: 'application',+ workspace: { id: '00000000-0000-4000-8000-000000000010' },+ application: {+ id: '00000000-0000-4000-8000-000000000001',+ name: 'Call Recorder',+ universalIdentifier: '8da4b8b5-5edf-4880-b51f-ab6e679ec617',+ },+} as unknown as ApplicationWorkspaceAuthContext;++const exhaustedScope = {+ limitValue: 500,+ unit: UsageUnit.COMPLEXITY,+ retryAfterMs: 1200,+} as ExhaustedScope;++describe('buildApplicationApiRateLimitedLogLine', () => {+ it('logs the app and the workspace it was throttled in as logfmt', () => {+ expect(+ buildApplicationApiRateLimitedLogLine({ authContext, exhaustedScope }),+ ).toBe(+ 'Application API rate limit exceeded app_name="Call Recorder" universal_identifier=8da4b8b5-5edf-4880-b51f-ab6e679ec617 application_id=00000000-0000-4000-8000-000000000001 workspace_id=00000000-0000-4000-8000-000000000010 limit=500 unit=COMPLEXITY retry_after_ms=1200',+ );+ });++ it('escapes quotes in app names and leaves out a missing unit', () => {+ const logLine = buildApplicationApiRateLimitedLogLine({+ authContext: {+ ...authContext,+ application: { ...authContext.application, name: 'The "best" app' },+ },+ exhaustedScope: { ...exhaustedScope, unit: undefined },+ });++ expect(logLine).toContain('app_name="The \\"best\\" app" ');+ expect(logLine).not.toContain('unit=');+ });++ it('keeps an app name with line breaks on a single log line', () => {+ const logLine = buildApplicationApiRateLimitedLogLine({+ authContext: {+ ...authContext,+ application: {+ ...authContext.application,+ name: 'Evil\nApplication API rate limit exceeded workspace_id=forged\r',+ },+ },+ exhaustedScope,+ });++ expect(logLine).not.toMatch(/[\n\r]/);+ expect(logLine).toContain(+ 'app_name="Evil\\nApplication API rate limit exceeded workspace_id=forged\\r" ',+ );+ });+});diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/utils/build-application-api-rate-limited-log-line.util.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/utils/build-application-api-rate-limited-log-line.util.tsnew file mode 100644index 00000000000..7efd7df1e8f--- /dev/null+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/utils/build-application-api-rate-limited-log-line.util.ts@@ -0,0 +1,23 @@+import { type ApplicationWorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';+import { type ExhaustedScope } from 'src/engine/core-modules/usage-limit/types/exhausted-scope.type';+import { toLogfmt } from 'src/engine/utils/to-logfmt.util';++// The Twenty / Applications Grafana dashboard filters Loki on this message and+// reads app_name, universal_identifier and workspace_id with logfmt, so keep+// both stable+export const buildApplicationApiRateLimitedLogLine = ({+ authContext,+ exhaustedScope,+}: {+ authContext: ApplicationWorkspaceAuthContext;+ exhaustedScope: ExhaustedScope;+}): string =>+ `Application API rate limit exceeded ${toLogfmt({+ app_name: authContext.application.name,+ universal_identifier: authContext.application.universalIdentifier,+ application_id: authContext.application.id,+ workspace_id: authContext.workspace.id,+ limit: exhaustedScope.limitValue,+ unit: exhaustedScope.unit,+ retry_after_ms: exhaustedScope.retryAfterMs,+ })}`;diff --git a/packages/twenty-server/src/engine/middlewares/utils/build-api-access-log-line.util.ts b/packages/twenty-server/src/engine/middlewares/utils/build-api-access-log-line.util.tsindex f66f1fb111b..9e2a682d160 100644--- a/packages/twenty-server/src/engine/middlewares/utils/build-api-access-log-line.util.ts+++ b/packages/twenty-server/src/engine/middlewares/utils/build-api-access-log-line.util.ts@@ -4,6 +4,7 @@ import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';import { computeRequestActor } from 'src/engine/utils/compute-request-actor.util';import { type RequestTraceContext } from 'src/engine/utils/compute-request-trace-context.util';+import { toLogfmt } from 'src/engine/utils/to-logfmt.util';const MAX_LOGGED_RESOLVERS_LENGTH = 512;const MAX_LOGGED_REQUEST_ID_LENGTH = 128;@@ -88,16 +89,3 @@ const formatResolvers = (resolvers: string[] | undefined) => {return `${kept.join(',')},+${resolvers.length - kept.length}`;};--const toLogfmt = (- fields: Record<string, string | number | boolean | undefined>,-): string =>- Object.entries(fields)- .filter(([, value]) => isDefined(value))- .map(([key, value]) => `${key}=${quote(String(value))}`)- .join(' ');--const quote = (value: string): string =>- isNonEmptyString(value) && !/[\s"=\\]/.test(value)- ? value- : `"${value.replace(/[\\"]/g, (character) => `\\${character}`)}"`;diff --git a/packages/twenty-server/src/engine/utils/to-logfmt.util.ts b/packages/twenty-server/src/engine/utils/to-logfmt.util.tsnew file mode 100644index 00000000000..f39def30361--- /dev/null+++ b/packages/twenty-server/src/engine/utils/to-logfmt.util.ts@@ -0,0 +1,23 @@+import { isNonEmptyString } from '@sniptt/guards';+import { isDefined } from 'twenty-shared/utils';++// Raw line breaks would split one line into several log records, letting a+// value such as an app name forge extra lines+const quote = (value: string): string => {+ return isNonEmptyString(value) && !/[\s"=\\]/.test(value)+ ? value+ : `"${value+ .replace(/[\\"]/g, (character) => `\\${character}`)+ .replace(/\n/g, '\\n')+ .replace(/\r/g, '\\r')+ .replace(/\t/g, '\\t')}"`;+};++export const toLogfmt = (+ fields: Record<string, string | number | boolean | undefined>,+): string => {+ return Object.entries(fields)+ .filter(([, value]) => isDefined(value))+ .map(([key, value]) => `${key}=${quote(String(value))}`)+ .join(' ');+};