The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Add GraphQL direct execution cost metrics (#23655)
Weiko6802eb3adfbb
diff --git a/packages/twenty-server/src/engine/api/graphql/direct-execution/direct-execution.service.ts b/packages/twenty-server/src/engine/api/graphql/direct-execution/direct-execution.service.tsindex 42e9e7b00af..0f106d6b2a8 100644--- a/packages/twenty-server/src/engine/api/graphql/direct-execution/direct-execution.service.ts+++ b/packages/twenty-server/src/engine/api/graphql/direct-execution/direct-execution.service.ts@@ -1,6 +1,10 @@+import { createHash } from 'crypto';+import { Injectable } from '@nestjs/common';import { type MessageDescriptor } from '@lingui/core';+import { type Histogram } from '@opentelemetry/api';+import * as Sentry from '@sentry/node';import { type Request } from 'express';import {GraphQLError,@@ -37,6 +41,7 @@ import { assertUpdateManyArgs } from 'src/engine/api/graphql/direct-execution/utimport { assertUpdateOneArgs } from 'src/engine/api/graphql/direct-execution/utils/assert-update-one-args.util';import { type ResolverNameMapEntry } from 'src/engine/api/graphql/direct-execution/utils/build-resolver-name-map.util';import { buildWorkspaceSchemaBuilderContext } from 'src/engine/api/graphql/direct-execution/utils/build-workspace-schema-builder-context.util';+import { computeGraphQLDirectExecutionQueryCost } from 'src/engine/api/graphql/direct-execution/utils/compute-graphql-direct-execution-query-cost.util';import { extractArgumentsFromAst } from 'src/engine/api/graphql/direct-execution/utils/extract-arguments-from-ast.util';import { graphQLBuildFragmentMap } from 'src/engine/api/graphql/direct-execution/utils/graphql-build-fragment-map.util';import { graphQLBuildPartialResolveInfo } from 'src/engine/api/graphql/direct-execution/utils/graphql-build-partial-resolve-info.util';@@ -76,6 +81,12 @@ type DirectExecutionResult = {errors?: GraphQLFormattedError[];};+const QUERY_COST_BUCKETS = [+ 10, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 25_000, 50_000, 100_000,+ 250_000,+];+const QUERY_FINGERPRINT_MIN_COST = 5000;+@Injectable()export class DirectExecutionService {private readonly factoryMap: Map<@@ -85,6 +96,8 @@ export class DirectExecutionService {private readonly argsAssertionMap: Map<string, (args: unknown) => void>;+ private readonly queryCostHistogram: Histogram;+constructor(private readonly workspaceFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService,private readonly workspaceCacheService: WorkspaceCacheService,@@ -108,6 +121,17 @@ export class DirectExecutionService {private readonly restoreManyResolverFactory: RestoreManyResolverFactory,private readonly mergeManyResolverFactory: MergeManyResolverFactory,) {+ this.queryCostHistogram = this.metricsService+ .getMeter()+ .createHistogram(+ 'twenty_graphql_direct_execution_estimated_result_field_count',+ {+ description:+ 'Estimated number of result field values produced by a directly executed GraphQL request',+ advice: { explicitBucketBoundaries: QUERY_COST_BUCKETS },+ },+ );+this.factoryMap = new Map<string, WorkspaceResolverBuilderFactoryInterface>([[RESOLVER_METHOD_NAMES.FIND_MANY, this.findManyResolverFactory],@@ -217,6 +241,14 @@ export class DirectExecutionService {const { idByNameSingular: objectIdByNameSingular } =buildObjectIdByNameMaps(flatObjectMetadataMaps);+ this.recordQueryCost({+ document,+ fragmentMap,+ topLevelFields,+ variables,+ graphQLResolverNameMap,+ });+const errors: GraphQLFormattedError[] = [];await Promise.all(@@ -410,6 +442,59 @@ export class DirectExecutionService {};}+ private recordQueryCost({+ document,+ fragmentMap,+ topLevelFields,+ variables,+ graphQLResolverNameMap,+ }: {+ document: DocumentNode;+ fragmentMap: ReturnType<typeof graphQLBuildFragmentMap>;+ topLevelFields: FieldNode[];+ variables: Record<string, unknown>;+ graphQLResolverNameMap: Record<string, ResolverNameMapEntry>;+ }): void {+ const queryCost = computeGraphQLDirectExecutionQueryCost({+ rootFields: topLevelFields.flatMap((field) => {+ const entry = graphQLResolverNameMap[field.name.value];++ if (!isDefined(entry)) {+ return [];+ }++ return [+ {+ field,+ method: entry.method,+ args: extractArgumentsFromAst(field.arguments, variables),+ },+ ];+ }),+ fragmentMap,+ });++ this.queryCostHistogram.record(queryCost.estimatedResultFieldCount);++ const query = document.loc?.source.body;++ Sentry.getActiveSpan()?.setAttributes({+ 'graphql.direct_execution.estimated_result_field_count':+ queryCost.estimatedResultFieldCount,+ ...(isDefined(query) &&+ queryCost.estimatedResultFieldCount >= QUERY_FINGERPRINT_MIN_COST && {+ 'graphql.direct_execution.query_fingerprint': createHash('sha256')+ .update(query)+ .digest('hex')+ .slice(0, 16),+ }),+ 'graphql.direct_execution.requested_row_count':+ queryCost.requestedRowCount,+ 'graphql.direct_execution.selected_leaf_field_count':+ queryCost.selectedLeafFieldCount,+ });+ }+private checkRootResolverLimitsOrThrow(topLevelFields: FieldNode[]): void {const maxRootResolvers = this.twentyConfigService.get('GRAPHQL_MAX_ROOT_RESOLVERS',diff --git a/packages/twenty-server/src/engine/api/graphql/direct-execution/utils/__tests__/compute-graphql-direct-execution-query-cost.util.spec.ts b/packages/twenty-server/src/engine/api/graphql/direct-execution/utils/__tests__/compute-graphql-direct-execution-query-cost.util.spec.tsnew file mode 100644index 00000000000..bb68535c10c--- /dev/null+++ b/packages/twenty-server/src/engine/api/graphql/direct-execution/utils/__tests__/compute-graphql-direct-execution-query-cost.util.spec.ts@@ -0,0 +1,179 @@+import { parse } from 'graphql';++import { computeGraphQLDirectExecutionQueryCost } from 'src/engine/api/graphql/direct-execution/utils/compute-graphql-direct-execution-query-cost.util';+import { extractArgumentsFromAst } from 'src/engine/api/graphql/direct-execution/utils/extract-arguments-from-ast.util';+import { graphQLBuildFragmentMap } from 'src/engine/api/graphql/direct-execution/utils/graphql-build-fragment-map.util';+import { graphQLExtractTopLevelFields } from 'src/engine/api/graphql/direct-execution/utils/graphql-extract-top-level-fields.util';+import { RESOLVER_METHOD_NAMES } from 'src/engine/api/graphql/workspace-resolver-builder/constants/resolver-method-names';++describe('computeGraphQLDirectExecutionQueryCost', () => {+ it('multiplies selected fields by the requested row count', () => {+ const document = parse(`+ query FindPeople($first: Int!) {+ people(first: $first) {+ edges {+ node {+ id+ name {+ firstName+ lastName+ }+ }+ cursor+ }+ pageInfo {+ hasNextPage+ }+ }+ }+ `);+ const variables = { first: 25 };+ const [field] = graphQLExtractTopLevelFields(document, 'FindPeople');++ const result = computeGraphQLDirectExecutionQueryCost({+ rootFields: [+ {+ field,+ method: RESOLVER_METHOD_NAMES.FIND_MANY,+ args: extractArgumentsFromAst(field.arguments, variables),+ },+ ],+ fragmentMap: graphQLBuildFragmentMap(document),+ });++ expect(result).toEqual({+ estimatedResultFieldCount: 125,+ requestedRowCount: 25,+ selectedLeafFieldCount: 5,+ });+ });++ it('uses the maximum page size when find many has no explicit limit', () => {+ const document = parse(`+ query FindPeople {+ people {+ edges {+ node {+ id+ name {+ firstName+ }+ }+ }+ }+ }+ `);+ const [field] = graphQLExtractTopLevelFields(document, 'FindPeople');++ const result = computeGraphQLDirectExecutionQueryCost({+ rootFields: [+ {+ field,+ method: RESOLVER_METHOD_NAMES.FIND_MANY,+ args: {},+ },+ ],+ fragmentMap: graphQLBuildFragmentMap(document),+ });++ expect(result).toEqual({+ estimatedResultFieldCount: 400,+ requestedRowCount: 200,+ selectedLeafFieldCount: 2,+ });+ });++ it('counts fragment fields and combines root resolver costs', () => {+ const document = parse(`+ query FindRecords {+ people(first: 10) {+ edges {+ node {+ ...PersonFields+ }+ }+ }+ findOneCompany(filter: { id: { eq: "company-id" } }) {+ id+ name+ }+ }++ fragment PersonFields on Person {+ id+ jobTitle+ }+ `);+ const fields = graphQLExtractTopLevelFields(document, 'FindRecords');++ const result = computeGraphQLDirectExecutionQueryCost({+ rootFields: [+ {+ field: fields[0],+ method: RESOLVER_METHOD_NAMES.FIND_MANY,+ args: extractArgumentsFromAst(fields[0].arguments, {}),+ },+ {+ field: fields[1],+ method: RESOLVER_METHOD_NAMES.FIND_ONE,+ args: extractArgumentsFromAst(fields[1].arguments, {}),+ },+ ],+ fragmentMap: graphQLBuildFragmentMap(document),+ });++ expect(result).toEqual({+ estimatedResultFieldCount: 22,+ requestedRowCount: 11,+ selectedLeafFieldCount: 4,+ });+ });++ it('computes each shared fragment once', () => {+ const fragmentCount = 60;+ const fragmentDefinitions = Array.from(+ { length: fragmentCount },+ (_, index) =>+ index === fragmentCount - 1+ ? `fragment SharedFragment${index} on Person { id }`+ : `fragment SharedFragment${index} on Person {+ ...SharedFragment${index + 1}+ ...SharedFragment${index + 1}+ }`,+ ).join('\n');+ const document = parse(`+ query FindPeople {+ people(first: 1) {+ edges {+ node {+ ...SharedFragment0+ }+ }+ }+ }++ ${fragmentDefinitions}+ `);+ const [field] = graphQLExtractTopLevelFields(document, 'FindPeople');+ const fragmentMap = graphQLBuildFragmentMap(document);+ const fragmentLookupSpy = jest.spyOn(fragmentMap, 'get');++ const result = computeGraphQLDirectExecutionQueryCost({+ rootFields: [+ {+ field,+ method: RESOLVER_METHOD_NAMES.FIND_MANY,+ args: { first: 1 },+ },+ ],+ fragmentMap,+ });++ expect(result).toEqual({+ estimatedResultFieldCount: Number.MAX_SAFE_INTEGER,+ requestedRowCount: 1,+ selectedLeafFieldCount: Number.MAX_SAFE_INTEGER,+ });+ expect(fragmentLookupSpy).toHaveBeenCalledTimes(fragmentCount);+ });+});diff --git a/packages/twenty-server/src/engine/api/graphql/direct-execution/utils/compute-graphql-direct-execution-query-cost.util.ts b/packages/twenty-server/src/engine/api/graphql/direct-execution/utils/compute-graphql-direct-execution-query-cost.util.tsnew file mode 100644index 00000000000..f438b0cb7c7--- /dev/null+++ b/packages/twenty-server/src/engine/api/graphql/direct-execution/utils/compute-graphql-direct-execution-query-cost.util.ts@@ -0,0 +1,174 @@+import {+ type FieldNode,+ type FragmentDefinitionNode,+ Kind,+ type SelectionNode,+} from 'graphql';+import { QUERY_MAX_RECORDS } from 'twenty-shared/constants';+import { isDefined } from 'twenty-shared/utils';++import { RESOLVER_METHOD_NAMES } from 'src/engine/api/graphql/workspace-resolver-builder/constants/resolver-method-names';+import { type WorkspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';++type DirectExecutionRootField = {+ field: FieldNode;+ method: WorkspaceResolverBuilderMethodNames;+ args: Record<string, unknown>;+};++export type GraphQLDirectExecutionQueryCost = {+ estimatedResultFieldCount: number;+ requestedRowCount: number;+ selectedLeafFieldCount: number;+};++type FragmentTraversalContext = {+ fieldCountByFragmentName: Map<string, number>;+ visitingFragmentNames: Set<string>;+};++const addWithMaximumSafeInteger = (left: number, right: number): number =>+ Math.min(left + right, Number.MAX_SAFE_INTEGER);++const multiplyWithMaximumSafeInteger = (left: number, right: number): number =>+ Math.min(left * right, Number.MAX_SAFE_INTEGER);++const countSelectedFields = (+ selections: readonly SelectionNode[],+ fragmentMap: Map<string, FragmentDefinitionNode>,+ fragmentTraversalContext: FragmentTraversalContext,+): number => {+ let fieldCount = 0;++ for (const selection of selections) {+ if (selection.kind === Kind.FIELD) {+ fieldCount = addWithMaximumSafeInteger(+ fieldCount,+ isDefined(selection.selectionSet)+ ? countSelectedFields(+ selection.selectionSet.selections,+ fragmentMap,+ fragmentTraversalContext,+ )+ : 1,+ );++ continue;+ }++ if (selection.kind === Kind.INLINE_FRAGMENT) {+ fieldCount = addWithMaximumSafeInteger(+ fieldCount,+ countSelectedFields(+ selection.selectionSet.selections,+ fragmentMap,+ fragmentTraversalContext,+ ),+ );++ continue;+ }++ const fragmentName = selection.name.value;+ const cachedFieldCount =+ fragmentTraversalContext.fieldCountByFragmentName.get(fragmentName);++ if (isDefined(cachedFieldCount)) {+ fieldCount = addWithMaximumSafeInteger(fieldCount, cachedFieldCount);++ continue;+ }++ if (fragmentTraversalContext.visitingFragmentNames.has(fragmentName)) {+ continue;+ }++ const fragment = fragmentMap.get(fragmentName);++ if (!isDefined(fragment)) {+ continue;+ }++ fragmentTraversalContext.visitingFragmentNames.add(fragmentName);++ const fragmentFieldCount = countSelectedFields(+ fragment.selectionSet.selections,+ fragmentMap,+ fragmentTraversalContext,+ );++ fragmentTraversalContext.visitingFragmentNames.delete(fragmentName);+ fragmentTraversalContext.fieldCountByFragmentName.set(+ fragmentName,+ fragmentFieldCount,+ );++ fieldCount = addWithMaximumSafeInteger(fieldCount, fragmentFieldCount);+ }++ return fieldCount;+};++const getRequestedRowCount = ({+ method,+ args,+}: Pick<DirectExecutionRootField, 'method' | 'args'>): number => {+ if (method !== RESOLVER_METHOD_NAMES.FIND_MANY) {+ return 1;+ }++ const requestedRowCount = args.first ?? args.last;++ return typeof requestedRowCount === 'number'+ ? Math.max(requestedRowCount, 0)+ : QUERY_MAX_RECORDS;+};++export const computeGraphQLDirectExecutionQueryCost = ({+ rootFields,+ fragmentMap,+}: {+ rootFields: DirectExecutionRootField[];+ fragmentMap: Map<string, FragmentDefinitionNode>;+}): GraphQLDirectExecutionQueryCost => {+ const fragmentTraversalContext: FragmentTraversalContext = {+ fieldCountByFragmentName: new Map(),+ visitingFragmentNames: new Set(),+ };++ return rootFields.reduce<GraphQLDirectExecutionQueryCost>(+ (queryCost, { field, method, args }) => {+ const selectedLeafFieldCount = isDefined(field.selectionSet)+ ? countSelectedFields(+ field.selectionSet.selections,+ fragmentMap,+ fragmentTraversalContext,+ )+ : 1;+ const requestedRowCount = getRequestedRowCount({ method, args });++ return {+ estimatedResultFieldCount: addWithMaximumSafeInteger(+ queryCost.estimatedResultFieldCount,+ multiplyWithMaximumSafeInteger(+ selectedLeafFieldCount,+ requestedRowCount,+ ),+ ),+ requestedRowCount: addWithMaximumSafeInteger(+ queryCost.requestedRowCount,+ requestedRowCount,+ ),+ selectedLeafFieldCount: addWithMaximumSafeInteger(+ queryCost.selectedLeafFieldCount,+ selectedLeafFieldCount,+ ),+ };+ },+ {+ estimatedResultFieldCount: 0,+ requestedRowCount: 0,+ selectedLeafFieldCount: 0,+ },+ );+};