The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(ai-tool): make search_output a raw-text occurrence search (#22034)
Etienne0f4c4e69a9d1
diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.jsonindex 926c93947b0..29b3fba6417 100644--- a/packages/twenty-server/package.json+++ b/packages/twenty-server/package.json@@ -152,7 +152,6 @@"pluralize": "8.0.0","postal-mime": "^2.6.1","psl": "^1.9.0",- "re2": "^1.25.0","react": "^19.2.0","react-dom": "^19.2.0","redis": "^4.7.0",diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-context-chars.constant.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-context-chars.constant.tsnew file mode 100644index 00000000000..9675e95963d--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-context-chars.constant.ts@@ -0,0 +1 @@+export const DEFAULT_SEARCH_OUTPUT_CONTEXT_CHARS = 100;diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-context-lines.constant.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-context-lines.constant.tsdeleted file mode 100644index fc793d89879..00000000000--- a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-context-lines.constant.ts+++ /dev/null@@ -1 +0,0 @@-export const DEFAULT_SEARCH_OUTPUT_CONTEXT_LINES = 2;diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-line-length.constant.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-line-length.constant.tsdeleted file mode 100644index 15a9414cded..00000000000--- a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-line-length.constant.ts+++ /dev/null@@ -1 +0,0 @@-export const SEARCH_OUTPUT_MAX_LINE_LENGTH = 500;diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-match-length.constant.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-match-length.constant.tsnew file mode 100644index 00000000000..24d53876469--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-match-length.constant.ts@@ -0,0 +1 @@+export const SEARCH_OUTPUT_MAX_MATCH_LENGTH = 500;diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/search-output-tool.schema.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/search-output-tool.schema.tsindex 990494b3d2e..887bf093cc6 100644--- a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/search-output-tool.schema.ts+++ b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/search-output-tool.schema.ts@@ -8,8 +8,9 @@ export const SearchOutputInputZodSchema = z.object({),pattern: z.string()+ .min(1).describe(- 'Text or regular expression to search for. Matched line by line against the indented JSON representation of the file.',+ 'Text or regular expression to search for. Treated as a regular expression when valid, otherwise as a literal string. Matched against the raw file text.',),maxMatches: z.number()@@ -17,22 +18,22 @@ export const SearchOutputInputZodSchema = z.object({.positive().max(100).optional()- .describe('Maximum number of matches to return (default 10).'),+ .describe('Maximum number of occurrences to return (default 10).'),offset: z.number().int().nonnegative().optional().describe(- 'Number of matches to skip before returning results, for stateless pagination (default 0).',+ 'Number of occurrences to skip before returning results, for stateless pagination (default 0).',),- contextLines: z+ contextChars: z.number().int().nonnegative()- .max(10)+ .max(2000).optional().describe(- 'Lines of context to include before and after each match (default 2).',+ 'Number of characters of context to include before and after each occurrence (default 100).',),});diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/search-output-tool.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/search-output-tool.tsindex e056e765ae0..1f68fe481d8 100644--- a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/search-output-tool.ts+++ b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/search-output-tool.ts@@ -3,7 +3,7 @@ import { Injectable, Logger } from '@nestjs/common';import { FileFolder } from 'twenty-shared/types';import { FileService } from 'src/engine/core-modules/file/services/file.service';-import { DEFAULT_SEARCH_OUTPUT_CONTEXT_LINES } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-context-lines.constant';+import { DEFAULT_SEARCH_OUTPUT_CONTEXT_CHARS } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-context-chars.constant';import { DEFAULT_SEARCH_OUTPUT_MAX_MATCHES } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/default-search-output-max-matches.constant';import { SearchOutputInputZodSchema } from 'src/engine/core-modules/tool/tools/output-navigation-tool/search-output-tool.schema';import { searchOutput } from 'src/engine/core-modules/tool/tools/output-navigation-tool/utils/search-output.util';@@ -13,20 +13,12 @@ import { type ToolOutput } from 'src/engine/core-modules/tool/types/tool-output.import { type Tool } from 'src/engine/core-modules/tool/types/tool.type';import { isDefined } from 'twenty-shared/utils';-type SearchOutputInput = {- fileId: string;- pattern: string;- maxMatches?: number;- offset?: number;- contextLines?: number;-};-@Injectable()export class SearchOutputTool implements Tool {private readonly logger = new Logger(SearchOutputTool.name);description =- 'Search (grep-like) within a large spilled tool output for a text or regex pattern, returning matching lines with surrounding context. Supports stateless pagination via offset. Use this to locate an error message or key in a file too large to inline.';+ 'Search (grep -o style) within a large spilled tool output for a text or regex pattern, returning every occurrence with surrounding characters of context. Works on raw text of any shape (CSV, HTML, stringified JSON, plain text), including single-line content. Supports stateless pagination via offset. Use this to locate an error message or key in a file too large to inline.';inputSchema = SearchOutputInputZodSchema;@@ -37,8 +29,23 @@ export class SearchOutputTool implements Tool {context: ToolExecutionContext,): Promise<ToolOutput> {const { workspaceId } = context;- const { fileId, pattern, maxMatches, offset, contextLines } =- parameters as SearchOutputInput;++ const parseResult = SearchOutputInputZodSchema.safeParse(parameters);++ if (!parseResult.success) {+ return {+ success: false,+ message: 'Invalid input for search output',+ error: parseResult.error.issues+ .map(+ (issue) => `${issue.path.join('.') || '(root)'}: ${issue.message}`,+ )+ .join('; '),+ };+ }++ const { fileId, pattern, maxMatches, offset, contextChars } =+ parseResult.data;let fileContent: { buffer: Buffer; mimeType: string } | null;@@ -66,22 +73,14 @@ export class SearchOutputTool implements Tool {};}- const rawContent = fileContent.buffer.toString('utf-8');-- let content: string;-- try {- content = JSON.stringify(JSON.parse(rawContent), null, 2);- } catch {- content = rawContent;- }+ const content = fileContent.buffer.toString('utf-8');const result = searchOutput({content,pattern,maxMatches: maxMatches ?? DEFAULT_SEARCH_OUTPUT_MAX_MATCHES,offset: offset ?? 0,- contextLines: contextLines ?? DEFAULT_SEARCH_OUTPUT_CONTEXT_LINES,+ contextChars: contextChars ?? DEFAULT_SEARCH_OUTPUT_CONTEXT_CHARS,});return {@@ -89,7 +88,7 @@ export class SearchOutputTool implements Tool {message:result.totalMatches === 0? `No matches for "${pattern}"`- : `Found ${result.totalMatches} match(es) for "${pattern}"`,+ : `Found ${result.totalMatches} occurrence(s) for "${pattern}"`,result,};}diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/utils/__tests__/search-output.util.spec.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/utils/__tests__/search-output.util.spec.tsindex 3e3abcda26a..b1c529a4f03 100644--- a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/utils/__tests__/search-output.util.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/utils/__tests__/search-output.util.spec.ts@@ -1,6 +1,7 @@+import { SEARCH_OUTPUT_MAX_MATCH_LENGTH } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-match-length.constant';import { searchOutput } from 'src/engine/core-modules/tool/tools/output-navigation-tool/utils/search-output.util';-const content = [+const multiLineContent = ['line 0 ok','line 1 error: boom','line 2 ok',@@ -10,31 +11,53 @@ const content = [].join('\n');describe('searchOutput', () => {- it('returns matches with context and line numbers', () => {+ it('returns each occurrence with a character window and offset', () => {const result = searchOutput({- content,+ content: multiLineContent,pattern: 'error',maxMatches: 10,offset: 0,- contextLines: 1,+ contextChars: 5,});expect(result.totalMatches).toBe(3);expect(result.hasMore).toBe(false);- expect(result.matches[0]).toEqual({- lineNumber: 2,- match: 'line 1 error: boom',- context: '1: line 0 ok\n2: line 1 error: boom\n3: line 2 ok',+ expect(result.matches).toHaveLength(3);+ expect(result.matches[0].match).toBe('error');+ expect(result.matches[0].charOffset).toBe(+ multiLineContent.indexOf('error'),+ );+ expect(result.matches[0].context).toContain('error');+ expect(result.matches[0].context.endsWith('…')).toBe(true);+ });++ it('finds every occurrence on a single newline-free line', () => {+ const compact = '{"a":"error","b":"error","c":"error"}';++ const result = searchOutput({+ content: compact,+ pattern: 'error',+ maxMatches: 10,+ offset: 0,+ contextChars: 0,});++ expect(result.totalMatches).toBe(3);+ expect(result.matches).toHaveLength(3);+ expect(result.matches.map((match) => match.charOffset)).toEqual([+ compact.indexOf('error'),+ compact.indexOf('error', compact.indexOf('error') + 1),+ compact.lastIndexOf('error'),+ ]);});it('caps results at maxMatches and reports hasMore', () => {const result = searchOutput({- content,+ content: multiLineContent,pattern: 'error',maxMatches: 2,offset: 0,- contextLines: 0,+ contextChars: 0,});expect(result.totalMatches).toBe(3);@@ -44,26 +67,28 @@ describe('searchOutput', () => {it('paginates with offset', () => {const result = searchOutput({- content,+ content: multiLineContent,pattern: 'error',maxMatches: 2,offset: 2,- contextLines: 0,+ contextChars: 0,});expect(result.totalMatches).toBe(3);expect(result.matches).toHaveLength(1);- expect(result.matches[0].lineNumber).toBe(6);+ expect(result.matches[0].charOffset).toBe(+ multiLineContent.lastIndexOf('error'),+ );expect(result.hasMore).toBe(false);});it('returns an empty result for zero matches', () => {const result = searchOutput({- content,+ content: multiLineContent,pattern: 'nonexistent',maxMatches: 10,offset: 0,- contextLines: 2,+ contextChars: 2,});expect(result).toEqual({ matches: [], totalMatches: 0, hasMore: false });@@ -71,26 +96,69 @@ describe('searchOutput', () => {it('supports regex patterns', () => {const result = searchOutput({- content,+ content: multiLineContent,pattern: 'splat|boom',maxMatches: 10,offset: 0,- contextLines: 0,+ contextChars: 0,});expect(result.totalMatches).toBe(3);});+ it('terminates on a zero-width pattern instead of looping forever', () => {+ const result = searchOutput({+ content: 'abc',+ pattern: 'x*',+ maxMatches: 10,+ offset: 0,+ contextChars: 0,+ });++ expect(result.totalMatches).toBeGreaterThan(0);+ expect(result.matches.every((match) => match.match === '')).toBe(true);+ });+it('falls back to literal matching for invalid regex', () => {const result = searchOutput({- content: 'a (b c\nd e f',+ content: 'a (b c d e f',pattern: '(b',maxMatches: 10,offset: 0,- contextLines: 0,+ contextChars: 0,});expect(result.totalMatches).toBe(1);- expect(result.matches[0].match).toBe('a (b c');+ expect(result.matches[0].match).toBe('(b');+ });++ it('truncates an overly long single match with a centered ellipsis', () => {+ const longMatch = 'z'.repeat(SEARCH_OUTPUT_MAX_MATCH_LENGTH + 200);++ const result = searchOutput({+ content: longMatch,+ pattern: 'z+',+ maxMatches: 10,+ offset: 0,+ contextChars: 0,+ });++ expect(result.totalMatches).toBe(1);+ expect(result.matches[0].match).toContain('…');+ expect(result.matches[0].match.length).toBeLessThanOrEqual(+ SEARCH_OUTPUT_MAX_MATCH_LENGTH,+ );+ });++ it('throws for an empty pattern instead of matching everything', () => {+ expect(() =>+ searchOutput({+ content: multiLineContent,+ pattern: '',+ maxMatches: 10,+ offset: 0,+ contextChars: 0,+ }),+ ).toThrow('Search pattern must be a non-empty string.');});});diff --git a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/utils/search-output.util.ts b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/utils/search-output.util.tsindex 6bfafc3347c..edc86205900 100644--- a/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/utils/search-output.util.ts+++ b/packages/twenty-server/src/engine/core-modules/tool/tools/output-navigation-tool/utils/search-output.util.ts@@ -1,7 +1,10 @@-import { SEARCH_OUTPUT_MAX_LINE_LENGTH } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-line-length.constant';+import { isNonEmptyString } from '@sniptt/guards';++import { SEARCH_OUTPUT_MAX_MATCH_LENGTH } from 'src/engine/core-modules/tool/tools/output-navigation-tool/constants/search-output-max-match-length.constant';+import { isDefined } from 'twenty-shared/utils';export type SearchMatch = {- lineNumber: number;+ charOffset: number;match: string;context: string;};@@ -17,64 +20,95 @@ const escapeRegExp = (value: string): string =>const compilePattern = (pattern: string): RegExp => {try {- return new RegExp(pattern);+ return new RegExp(pattern, 'g');} catch {- return new RegExp(escapeRegExp(pattern));+ return new RegExp(escapeRegExp(pattern), 'g');}};-const truncateLine = (line: string): string =>- line.length > SEARCH_OUTPUT_MAX_LINE_LENGTH- ? `${line.slice(0, SEARCH_OUTPUT_MAX_LINE_LENGTH)}…`- : line;+const truncateMatch = (match: string): string => {+ if (match.length <= SEARCH_OUTPUT_MAX_MATCH_LENGTH) {+ return match;+ }++ const half = Math.floor((SEARCH_OUTPUT_MAX_MATCH_LENGTH - 1) / 2);++ return `${match.slice(0, half)}…${match.slice(match.length - half)}`;+};++const buildContext = ({+ content,+ index,+ length,+ contextChars,+}: {+ content: string;+ index: number;+ length: number;+ contextChars: number;+}): string => {+ const start = Math.max(0, index - contextChars);+ const end = Math.min(content.length, index + length + contextChars);++ const prefix = start > 0 ? '…' : '';+ const suffix = end < content.length ? '…' : '';++ return `${prefix}${content.slice(start, end)}${suffix}`;+};export const searchOutput = ({content,pattern,maxMatches,offset,- contextLines,+ contextChars,}: {content: string;pattern: string;maxMatches: number;offset: number;- contextLines: number;+ contextChars: number;}): SearchOutputResult => {- const lines = content.split('\n');+ if (!isNonEmptyString(pattern)) {+ throw new Error('Search pattern must be a non-empty string.');+ }+const regex = compilePattern(pattern);- const matchLineIndices: number[] = [];+ const matches: SearchMatch[] = [];+ let totalMatches = 0;- for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {- if (regex.test(lines[lineIndex])) {- matchLineIndices.push(lineIndex);- }- }+ let execResult = regex.exec(content);- const totalMatches = matchLineIndices.length;- const selected = matchLineIndices.slice(offset, offset + maxMatches);+ while (isDefined(execResult)) {+ const index = execResult.index;+ const matched = execResult[0];- const matches: SearchMatch[] = selected.map((lineIndex) => {- const start = Math.max(0, lineIndex - contextLines);- const end = Math.min(lines.length - 1, lineIndex + contextLines);+ if (totalMatches >= offset && matches.length < maxMatches) {+ matches.push({+ charOffset: index,+ match: truncateMatch(matched),+ context: buildContext({+ content,+ index,+ length: matched.length,+ contextChars,+ }),+ });+ }- const contextBlock: string[] = [];+ totalMatches += 1;- for (let cursor = start; cursor <= end; cursor++) {- contextBlock.push(`${cursor + 1}: ${truncateLine(lines[cursor])}`);+ if (matched.length === 0) {+ regex.lastIndex += 1;}- return {- lineNumber: lineIndex + 1,- match: truncateLine(lines[lineIndex]),- context: contextBlock.join('\n'),- };- });+ execResult = regex.exec(content);+ }return {matches,totalMatches,- hasMore: offset + selected.length < totalMatches,+ hasMore: offset + matches.length < totalMatches,};};diff --git a/yarn.lock b/yarn.lockindex 584fb56b2f9..88253e1b805 100644--- a/yarn.lock+++ b/yarn.lock@@ -36351,16 +36351,6 @@ __metadata:languageName: nodelinkType: hard-"install-artifact-from-github@npm:^1.6.0":- version: 1.6.0- resolution: "install-artifact-from-github@npm:1.6.0"- bin:- install-from-cache: bin/install-from-cache.js- save-to-github-cache: bin/save-to-github-cache.js- checksum: 10c0/e480c994a6230ba9c24b76e29908dcfb29977d3ee5b4a193b5c793139065aa0deb9bc5f475d11fd347c516b88ddb1a0f02935aa11fd1d3585761a4a4ccc1885c- languageName: node- linkType: hard-"internal-slot@npm:^1.1.0":version: 1.1.0resolution: "internal-slot@npm:1.1.0"@@ -42148,15 +42138,6 @@ __metadata:languageName: nodelinkType: hard-"nan@npm:^2.27.0":- version: 2.27.0- resolution: "nan@npm:2.27.0"- dependencies:- node-gyp: "npm:latest"- checksum: 10c0/38eb00b06e40f0c65b6a98d75795f17d651a8b7b52f03873dff6902d0053f12e7638d7f64fc52bda6c8f8ec454d69636e3988c8a9eb2bc749c2d5c255ba55f4c- languageName: node- linkType: hard-"nan@npm:^2.4.0":version: 2.26.2resolution: "nan@npm:2.26.2"@@ -42687,7 +42668,7 @@ __metadata:languageName: nodelinkType: hard-"node-gyp@npm:^13.0.0, node-gyp@npm:latest":+"node-gyp@npm:latest":version: 13.0.0resolution: "node-gyp@npm:13.0.0"dependencies:@@ -46361,17 +46342,6 @@ __metadata:languageName: nodelinkType: hard-"re2@npm:^1.25.0":- version: 1.25.0- resolution: "re2@npm:1.25.0"- dependencies:- install-artifact-from-github: "npm:^1.6.0"- nan: "npm:^2.27.0"- node-gyp: "npm:^13.0.0"- checksum: 10c0/a1b89e4d6c594d3fe675fc18f5ee3ce06ea83d66ebc5b3b2d66abf8c93c57757478969d1c934db4379ffba0575154c149d4168a567d192a936fced369d7c3ecf- languageName: node- linkType: hard-"react-aria@npm:^3.48.0":version: 3.49.0resolution: "react-aria@npm:3.49.0"@@ -52860,7 +52830,6 @@ __metadata:postal-mime: "npm:^2.6.1"prettier: "npm:^3.1.1"psl: "npm:^1.9.0"- re2: "npm:^1.25.0"react: "npm:^19.2.0"react-dom: "npm:^19.2.0"redis: "npm:^4.7.0"