Skip to content

The open alternative to Salesforce, designed for AI.

Log the app and workspace when an application hits the API rate limit (#27655)

martmull1d3000ce0025
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts
index 4d4263913c4..32f75fec59d 100644
--- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts
+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-base-query-runner.service.ts
@@ -1,4 +1,4 @@
-import { Inject, Injectable } from '@nestjs/common';
+import { Inject, Injectable, Logger } from '@nestjs/common';
import { QUERY_MAX_RECORDS_FROM_RELATION } from 'twenty-shared/constants';
import { FieldMetadataType, type ObjectRecord } from 'twenty-shared/types';
@@ -20,6 +20,7 @@ import {
CommonQueryRunnerExceptionCode,
} from 'src/engine/api/common/common-query-runners/errors/common-query-runner.exception';
import { STANDARD_ERROR_MESSAGE } from 'src/engine/api/common/common-query-runners/errors/standard-error-message.constant';
+import { buildApplicationApiRateLimitedLogLine } from 'src/engine/api/common/common-query-runners/utils/build-application-api-rate-limited-log-line.util';
import { buildMutationQueryBuilder } from 'src/engine/api/common/common-query-runners/utils/build-mutation-query-builder.util';
import { computeMaxFieldCountPerRecord } from 'src/engine/api/common/common-query-runners/utils/compute-max-field-count-per-record.util';
import { isRecordFilterEmpty } from 'src/engine/api/common/common-query-runners/utils/is-record-filter-empty.util';
@@ -73,6 +74,8 @@ export abstract class CommonBaseQueryRunnerService<
Args extends CommonQueryArgs,
Output extends CommonQueryResult,
> {
+ private readonly logger = new Logger(CommonBaseQueryRunnerService.name);
+
@Inject()
protected readonly workspaceQueryHookService: WorkspaceQueryHookService;
@Inject()
@@ -547,7 +550,7 @@ export abstract class CommonBaseQueryRunnerService<
isDefined(error.exhaustedScope) &&
error.exhaustedScope.isDefault
) {
- await this.incrementApiSpeedCeilingMetrics({
+ await this.reportApiSpeedCeilingHit({
authContext,
exhaustedScope: error.exhaustedScope,
});
@@ -557,7 +560,7 @@ export abstract class CommonBaseQueryRunnerService<
}
}
- private async incrementApiSpeedCeilingMetrics({
+ private async reportApiSpeedCeilingHit({
authContext,
exhaustedScope,
}: {
@@ -584,6 +587,10 @@ export abstract class CommonBaseQueryRunnerService<
source_type: authContext.application.sourceType,
},
});
+
+ this.logger.warn(
+ buildApplicationApiRateLimitedLogLine({ authContext, exhaustedScope }),
+ );
}
}
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/utils/__tests__/build-application-api-rate-limited-log-line.util.spec.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/utils/__tests__/build-application-api-rate-limited-log-line.util.spec.ts
new file mode 100644
index 00000000000..5c86674416a
--- /dev/null
+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/utils/__tests__/build-application-api-rate-limited-log-line.util.spec.ts
@@ -0,0 +1,61 @@
+import { buildApplicationApiRateLimitedLogLine } from 'src/engine/api/common/common-query-runners/utils/build-application-api-rate-limited-log-line.util';
+import { type ApplicationWorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';
+import { type ExhaustedScope } from 'src/engine/core-modules/usage-limit/types/exhausted-scope.type';
+import { UsageUnit } from 'src/engine/core-modules/usage/enums/usage-unit.enum';
+
+const authContext = {
+ type: 'application',
+ workspace: { id: '00000000-0000-4000-8000-000000000010' },
+ application: {
+ id: '00000000-0000-4000-8000-000000000001',
+ name: 'Call Recorder',
+ universalIdentifier: '8da4b8b5-5edf-4880-b51f-ab6e679ec617',
+ },
+} as unknown as ApplicationWorkspaceAuthContext;
+
+const exhaustedScope = {
+ limitValue: 500,
+ unit: UsageUnit.COMPLEXITY,
+ retryAfterMs: 1200,
+} as ExhaustedScope;
+
+describe('buildApplicationApiRateLimitedLogLine', () => {
+ it('logs the app and the workspace it was throttled in as logfmt', () => {
+ expect(
+ buildApplicationApiRateLimitedLogLine({ authContext, exhaustedScope }),
+ ).toBe(
+ 'Application API rate limit exceeded app_name="Call Recorder" universal_identifier=8da4b8b5-5edf-4880-b51f-ab6e679ec617 application_id=00000000-0000-4000-8000-000000000001 workspace_id=00000000-0000-4000-8000-000000000010 limit=500 unit=COMPLEXITY retry_after_ms=1200',
+ );
+ });
+
+ it('escapes quotes in app names and leaves out a missing unit', () => {
+ const logLine = buildApplicationApiRateLimitedLogLine({
+ authContext: {
+ ...authContext,
+ application: { ...authContext.application, name: 'The "best" app' },
+ },
+ exhaustedScope: { ...exhaustedScope, unit: undefined },
+ });
+
+ expect(logLine).toContain('app_name="The \\"best\\" app" ');
+ expect(logLine).not.toContain('unit=');
+ });
+
+ it('keeps an app name with line breaks on a single log line', () => {
+ const logLine = buildApplicationApiRateLimitedLogLine({
+ authContext: {
+ ...authContext,
+ application: {
+ ...authContext.application,
+ name: 'Evil\nApplication API rate limit exceeded workspace_id=forged\r',
+ },
+ },
+ exhaustedScope,
+ });
+
+ expect(logLine).not.toMatch(/[\n\r]/);
+ expect(logLine).toContain(
+ 'app_name="Evil\\nApplication API rate limit exceeded workspace_id=forged\\r" ',
+ );
+ });
+});
diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/utils/build-application-api-rate-limited-log-line.util.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/utils/build-application-api-rate-limited-log-line.util.ts
new file mode 100644
index 00000000000..7efd7df1e8f
--- /dev/null
+++ b/packages/twenty-server/src/engine/api/common/common-query-runners/utils/build-application-api-rate-limited-log-line.util.ts
@@ -0,0 +1,23 @@
+import { type ApplicationWorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';
+import { type ExhaustedScope } from 'src/engine/core-modules/usage-limit/types/exhausted-scope.type';
+import { toLogfmt } from 'src/engine/utils/to-logfmt.util';
+
+// The Twenty / Applications Grafana dashboard filters Loki on this message and
+// reads app_name, universal_identifier and workspace_id with logfmt, so keep
+// both stable
+export const buildApplicationApiRateLimitedLogLine = ({
+ authContext,
+ exhaustedScope,
+}: {
+ authContext: ApplicationWorkspaceAuthContext;
+ exhaustedScope: ExhaustedScope;
+}): string =>
+ `Application API rate limit exceeded ${toLogfmt({
+ app_name: authContext.application.name,
+ universal_identifier: authContext.application.universalIdentifier,
+ application_id: authContext.application.id,
+ workspace_id: authContext.workspace.id,
+ limit: exhaustedScope.limitValue,
+ unit: exhaustedScope.unit,
+ retry_after_ms: exhaustedScope.retryAfterMs,
+ })}`;
diff --git a/packages/twenty-server/src/engine/middlewares/utils/build-api-access-log-line.util.ts b/packages/twenty-server/src/engine/middlewares/utils/build-api-access-log-line.util.ts
index f66f1fb111b..9e2a682d160 100644
--- a/packages/twenty-server/src/engine/middlewares/utils/build-api-access-log-line.util.ts
+++ b/packages/twenty-server/src/engine/middlewares/utils/build-api-access-log-line.util.ts
@@ -4,6 +4,7 @@ import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';
import { computeRequestActor } from 'src/engine/utils/compute-request-actor.util';
import { type RequestTraceContext } from 'src/engine/utils/compute-request-trace-context.util';
+import { toLogfmt } from 'src/engine/utils/to-logfmt.util';
const MAX_LOGGED_RESOLVERS_LENGTH = 512;
const MAX_LOGGED_REQUEST_ID_LENGTH = 128;
@@ -88,16 +89,3 @@ const formatResolvers = (resolvers: string[] | undefined) => {
return `${kept.join(',')},+${resolvers.length - kept.length}`;
};
-
-const toLogfmt = (
- fields: Record<string, string | number | boolean | undefined>,
-): string =>
- Object.entries(fields)
- .filter(([, value]) => isDefined(value))
- .map(([key, value]) => `${key}=${quote(String(value))}`)
- .join(' ');
-
-const quote = (value: string): string =>
- isNonEmptyString(value) && !/[\s"=\\]/.test(value)
- ? value
- : `"${value.replace(/[\\"]/g, (character) => `\\${character}`)}"`;
diff --git a/packages/twenty-server/src/engine/utils/to-logfmt.util.ts b/packages/twenty-server/src/engine/utils/to-logfmt.util.ts
new file mode 100644
index 00000000000..f39def30361
--- /dev/null
+++ b/packages/twenty-server/src/engine/utils/to-logfmt.util.ts
@@ -0,0 +1,23 @@
+import { isNonEmptyString } from '@sniptt/guards';
+import { isDefined } from 'twenty-shared/utils';
+
+// Raw line breaks would split one line into several log records, letting a
+// value such as an app name forge extra lines
+const quote = (value: string): string => {
+ return isNonEmptyString(value) && !/[\s"=\\]/.test(value)
+ ? value
+ : `"${value
+ .replace(/[\\"]/g, (character) => `\\${character}`)
+ .replace(/\n/g, '\\n')
+ .replace(/\r/g, '\\r')
+ .replace(/\t/g, '\\t')}"`;
+};
+
+export const toLogfmt = (
+ fields: Record<string, string | number | boolean | undefined>,
+): string => {
+ return Object.entries(fields)
+ .filter(([, value]) => isDefined(value))
+ .map(([key, value]) => `${key}=${quote(String(value))}`)
+ .join(' ');
+};