Skip to content

The open alternative to Salesforce, designed for AI.

feat(ai-agent): suggest similar tool names when tool discovery misses (#21654)

Félix Malfait1e8169ca3e32
diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts
index 099e0579438..ad17ad572fd 100644
--- a/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/tool-provider/services/tool-registry.service.ts
@@ -13,6 +13,7 @@ import { type LearnToolsAspect } from 'src/engine/core-modules/tool-provider/too
import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/tool-context.type';
import { type ToolDescriptor } from 'src/engine/core-modules/tool-provider/types/tool-descriptor.type';
import { type ToolIndexEntry } from 'src/engine/core-modules/tool-provider/types/tool-index-entry.type';
+import { findSimilarToolNames } from 'src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util';
import { wrapWithErrorHandler } from 'src/engine/core-modules/tool-provider/utils/tool-error.util';
import { type ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.type';
import {
@@ -241,6 +242,31 @@ export class ToolRegistryService {
});
}
+ async suggestSimilarToolNames(
+ toolNames: string[],
+ context: ToolContext,
+ ): Promise<Record<string, string[]>> {
+ const fullContext = this.buildContextFromToolContext(context);
+
+ const catalog = await this.getCatalog(fullContext);
+ const candidateToolNames = catalog.map((entry) => entry.name);
+
+ const suggestionsByToolName: Record<string, string[]> = {};
+
+ for (const toolName of toolNames) {
+ const similarToolNames = findSimilarToolNames(
+ toolName,
+ candidateToolNames,
+ );
+
+ if (similarToolNames.length > 0) {
+ suggestionsByToolName[toolName] = similarToolNames;
+ }
+ }
+
+ return suggestionsByToolName;
+ }
+
async resolveAndExecute(
toolName: string,
args: Record<string, unknown> | undefined,
@@ -254,10 +280,19 @@ export class ToolRegistryService {
const entry = index.find((indexEntry) => indexEntry.name === toolName);
if (!entry) {
+ const similarToolNames = findSimilarToolNames(
+ toolName,
+ index.map((indexEntry) => indexEntry.name),
+ );
+ const suggestionHint =
+ similarToolNames.length > 0
+ ? ` Did you mean: ${similarToolNames.join(', ')}?`
+ : '';
+
return {
success: false,
message: `Tool "${toolName}" not found`,
- error: `Tool "${toolName}" not found. Use learn_tools to discover available tools.`,
+ error: `Tool "${toolName}" not found.${suggestionHint} Use learn_tools to discover available tools.`,
};
}
diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/tools/__tests__/learn-tools.tool.spec.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/tools/__tests__/learn-tools.tool.spec.ts
new file mode 100644
index 00000000000..14655aac883
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/tool-provider/tools/__tests__/learn-tools.tool.spec.ts
@@ -0,0 +1,105 @@
+import { type ToolRegistryService } from 'src/engine/core-modules/tool-provider/services/tool-registry.service';
+import { createLearnToolsTool } from 'src/engine/core-modules/tool-provider/tools/learn-tools.tool';
+import { type ToolContext } from 'src/engine/core-modules/tool-provider/types/tool-context.type';
+
+describe('createLearnToolsTool', () => {
+ const context = {} as ToolContext;
+
+ it('surfaces "did you mean" suggestions when a tool name is not found', async () => {
+ const toolRegistry = {
+ getToolInfo: jest.fn().mockResolvedValue([]),
+ suggestSimilarToolNames: jest
+ .fn()
+ .mockResolvedValue({ group_by_person: ['group_by_people'] }),
+ } as unknown as ToolRegistryService;
+
+ const learnTools = createLearnToolsTool(toolRegistry, context);
+
+ const result = await learnTools.execute({
+ toolNames: ['group_by_person'],
+ aspects: ['description', 'schema'],
+ });
+
+ expect(result.notFound).toEqual(['group_by_person']);
+ expect(result.suggestions).toEqual({
+ group_by_person: ['group_by_people'],
+ });
+ expect(result.message).toContain(
+ 'group_by_person (did you mean: group_by_people?)',
+ );
+ });
+
+ it('does not look up suggestions when every tool resolves', async () => {
+ const suggestSimilarToolNames = jest.fn();
+ const toolRegistry = {
+ getToolInfo: jest
+ .fn()
+ .mockResolvedValue([
+ { name: 'group_by_people', description: 'Group people' },
+ ]),
+ suggestSimilarToolNames,
+ } as unknown as ToolRegistryService;
+
+ const learnTools = createLearnToolsTool(toolRegistry, context);
+
+ const result = await learnTools.execute({
+ toolNames: ['group_by_people'],
+ aspects: ['description'],
+ });
+
+ expect(result.notFound).toEqual([]);
+ expect(result.suggestions).toBeUndefined();
+ expect(suggestSimilarToolNames).not.toHaveBeenCalled();
+ expect(result.message).toBe('Learned 1 tool: group_by_people.');
+ });
+
+ it('pluralizes the learned-tools count', async () => {
+ const toolRegistry = {
+ getToolInfo: jest
+ .fn()
+ .mockResolvedValue([
+ { name: 'find_many_people' },
+ { name: 'group_by_people' },
+ ]),
+ suggestSimilarToolNames: jest.fn(),
+ } as unknown as ToolRegistryService;
+
+ const learnTools = createLearnToolsTool(toolRegistry, context);
+
+ const result = await learnTools.execute({
+ toolNames: ['find_many_people', 'group_by_people'],
+ aspects: ['description'],
+ });
+
+ expect(result.message).toBe(
+ 'Learned 2 tools: find_many_people, group_by_people.',
+ );
+ });
+
+ it('does not report excluded tools as not found or suggest alternatives', async () => {
+ const suggestSimilarToolNames = jest.fn();
+ const toolRegistry = {
+ getToolInfo: jest.fn().mockResolvedValue([]),
+ suggestSimilarToolNames,
+ } as unknown as ToolRegistryService;
+
+ const learnTools = createLearnToolsTool(
+ toolRegistry,
+ context,
+ new Set(['code_interpreter']),
+ );
+
+ const result = await learnTools.execute({
+ toolNames: ['code_interpreter'],
+ aspects: ['description'],
+ });
+
+ expect(toolRegistry.getToolInfo).toHaveBeenCalledWith([], context, [
+ 'description',
+ ]);
+ expect(result.notFound).toEqual([]);
+ expect(result.suggestions).toBeUndefined();
+ expect(suggestSimilarToolNames).not.toHaveBeenCalled();
+ expect(result.message).toBe('No matching tools found.');
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/tools/learn-tools.tool.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/tools/learn-tools.tool.ts
index 4f314bb4871..47898ce7e2b 100644
--- a/packages/twenty-server/src/engine/core-modules/tool-provider/tools/learn-tools.tool.ts
+++ b/packages/twenty-server/src/engine/core-modules/tool-provider/tools/learn-tools.tool.ts
@@ -35,6 +35,7 @@ export type LearnToolsResultEntry = {
export type LearnToolsResult = {
tools: LearnToolsResultEntry[];
notFound: string[];
+ suggestions?: Record<string, string[]>;
message: string;
};
@@ -59,21 +60,49 @@ export const createLearnToolsTool = (
aspects,
);
- const foundNames = new Set(toolInfos.map((t) => t.name));
- const notFound = toolNames.filter((name) => !foundNames.has(name));
+ const foundNames = new Set(toolInfos.map((toolInfo) => toolInfo.name));
+ // Base notFound on allowedNames so excluded tools aren't surfaced as
+ // missing (which would also trigger misleading suggestions).
+ const notFound = allowedNames.filter((name) => !foundNames.has(name));
+
+ const suggestions: Record<string, string[]> =
+ notFound.length > 0
+ ? await toolRegistry.suggestSimilarToolNames(notFound, context)
+ : {};
+
+ const messageParts: string[] = [];
+
+ if (toolInfos.length > 0) {
+ const learnedNames = toolInfos.map((toolInfo) => toolInfo.name);
+ const toolNoun = learnedNames.length === 1 ? 'tool' : 'tools';
+
+ messageParts.push(
+ `Learned ${learnedNames.length} ${toolNoun}: ${learnedNames.join(', ')}`,
+ );
+ }
if (notFound.length > 0) {
- return {
- tools: toolInfos,
- notFound,
- message: `Learned ${toolInfos.length} tool(s). Could not find: ${notFound.join(', ')}.`,
- };
+ const notFoundDescription = notFound
+ .map((name) => {
+ const similarToolNames = suggestions[name];
+
+ return similarToolNames?.length
+ ? `${name} (did you mean: ${similarToolNames.join(', ')}?)`
+ : name;
+ })
+ .join('; ');
+
+ messageParts.push(`Could not find: ${notFoundDescription}`);
}
return {
tools: toolInfos,
- notFound: [],
- message: `Learned ${toolInfos.length} tool(s): ${toolInfos.map((t) => t.name).join(', ')}.`,
+ notFound,
+ ...(Object.keys(suggestions).length > 0 && { suggestions }),
+ message:
+ messageParts.length > 0
+ ? `${messageParts.join('. ')}.`
+ : 'No matching tools found.',
};
},
});
diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/find-similar-tool-names.util.spec.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/find-similar-tool-names.util.spec.ts
new file mode 100644
index 00000000000..7f548bdd914
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/__tests__/find-similar-tool-names.util.spec.ts
@@ -0,0 +1,59 @@
+import { findSimilarToolNames } from 'src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util';
+
+describe('findSimilarToolNames', () => {
+ const catalog = [
+ 'find_many_people',
+ 'find_one_person',
+ 'group_by_people',
+ 'group_by_companies',
+ 'create_one_person',
+ 'update_many_people',
+ ];
+
+ it('ranks the plural variant first for a singular group_by name', () => {
+ const suggestions = findSimilarToolNames('group_by_person', catalog);
+
+ expect(suggestions).toContain('group_by_people');
+ expect(suggestions[0]).toBe('group_by_people');
+ });
+
+ it('ranks the plural variant first for a singular find_many name', () => {
+ const suggestions = findSimilarToolNames('find_many_person', catalog);
+
+ expect(suggestions[0]).toBe('find_many_people');
+ });
+
+ it('ranks the closest candidate first for a typo', () => {
+ expect(findSimilarToolNames('group_by_peple', catalog)[0]).toBe(
+ 'group_by_people',
+ );
+ });
+
+ it('returns an empty array when nothing is close enough', () => {
+ expect(findSimilarToolNames('send_email', catalog)).toEqual([]);
+ });
+
+ it('never suggests an exact match', () => {
+ expect(findSimilarToolNames('group_by_people', catalog)).not.toContain(
+ 'group_by_people',
+ );
+ });
+
+ it('caps the number of suggestions at three', () => {
+ const manyCandidates = [
+ 'find_many_opportunities',
+ 'find_many_opportunity',
+ 'find_many_opportunitiez',
+ 'find_many_oportunities',
+ 'find_many_opportunitis',
+ ];
+
+ expect(
+ findSimilarToolNames('find_many_opportunites', manyCandidates),
+ ).toHaveLength(3);
+ });
+
+ it('handles an empty catalog', () => {
+ expect(findSimilarToolNames('group_by_person', [])).toEqual([]);
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util.ts b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util.ts
new file mode 100644
index 00000000000..52a73664bb0
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/tool-provider/utils/find-similar-tool-names.util.ts
@@ -0,0 +1,43 @@
+import { getEditDistance } from 'twenty-shared/workflow';
+
+const MAX_TOOL_NAME_SUGGESTIONS = 3;
+const PREFIX_MATCH_WEIGHT = 0.5;
+
+const getCommonPrefixLength = (left: string, right: string): number => {
+ const maxLength = Math.min(left.length, right.length);
+ let index = 0;
+
+ while (index < maxLength && left[index] === right[index]) {
+ index++;
+ }
+
+ return index;
+};
+
+export const findSimilarToolNames = (
+ toolName: string,
+ candidateToolNames: string[],
+): string[] =>
+ candidateToolNames
+ .map((candidate) => ({
+ candidate,
+ distance: getEditDistance(toolName, candidate),
+ prefixLength: getCommonPrefixLength(toolName, candidate),
+ }))
+ .filter(
+ ({ candidate, distance }) =>
+ distance > 0 && distance <= Math.ceil(candidate.length / 2),
+ )
+ // Reward a shared operation prefix so the same-operation candidate (the
+ // correct plural form) ranks ahead of a closer-but-different operation.
+ .sort((left, right) => {
+ const leftScore = left.distance - PREFIX_MATCH_WEIGHT * left.prefixLength;
+ const rightScore =
+ right.distance - PREFIX_MATCH_WEIGHT * right.prefixLength;
+
+ return (
+ leftScore - rightScore || left.candidate.localeCompare(right.candidate)
+ );
+ })
+ .slice(0, MAX_TOOL_NAME_SUGGESTIONS)
+ .map(({ candidate }) => candidate);