Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
fix(firecrawl): thread user query into summarizer prompt
rramprakashb32dde8f01bc
diff --git a/backend/pkg/tools/firecrawl.go b/backend/pkg/tools/firecrawl.goindex 7a4f9cc5..5db5817b 100644--- a/backend/pkg/tools/firecrawl.go+++ b/backend/pkg/tools/firecrawl.go@@ -201,10 +201,10 @@ func (f *firecrawl) search(ctx context.Context, query string, maxResults int) (s}defer resp.Body.Close()- return f.parseHTTPResponse(ctx, resp)+ return f.parseHTTPResponse(ctx, query, resp)}-func (f *firecrawl) parseHTTPResponse(ctx context.Context, resp *http.Response) (string, error) {+func (f *firecrawl) parseHTTPResponse(ctx context.Context, query string, resp *http.Response) (string, error) {switch resp.StatusCode {case http.StatusOK:var respBody firecrawlSearchResult@@ -217,7 +217,7 @@ func (f *firecrawl) parseHTTPResponse(ctx context.Context, resp *http.Response)}return "", fmt.Errorf("request failed")}- return f.buildFirecrawlResult(ctx, &respBody), nil+ return f.buildFirecrawlResult(ctx, query, &respBody), nilcase http.StatusBadRequest:return "", fmt.Errorf("request is invalid")case http.StatusUnauthorized:@@ -247,7 +247,7 @@ func (f *firecrawl) parseHTTPResponse(ctx context.Context, resp *http.Response)}}-func (f *firecrawl) buildFirecrawlResult(ctx context.Context, result *firecrawlSearchResult) string {+func (f *firecrawl) buildFirecrawlResult(ctx context.Context, query string, result *firecrawlSearchResult) string {var writer strings.Builderwriter.WriteString("# Links\n\n")@@ -264,7 +264,7 @@ func (f *firecrawl) buildFirecrawlResult(ctx context.Context, result *firecrawlS}if isMarkdownExists && f.summarizer != nil {- summarizePrompt, err := f.getSummarizePrompt(result)+ summarizePrompt, err := f.getSummarizePrompt(query, result)if err != nil {writer.WriteString(f.getContentFromResults(result.Data.Web))} else {@@ -305,10 +305,12 @@ type firecrawlPromptDoc struct {Markdown string}-func (f *firecrawl) getSummarizePrompt(result *firecrawlSearchResult) (string, error) {+func (f *firecrawl) getSummarizePrompt(query string, result *firecrawlSearchResult) (string, error) {templateText := `<instructions>TASK: Summarize web search results for the following user query:+USER QUERY: "{{.Query}}"+DATA:- <raw_content> tags contain web page content with attributes: id, title, url- Content may include HTML, structured data, tables, or plain text@@ -355,6 +357,7 @@ The summary MUST provide complete answers to the user's query, preserving all re}templateContext := map[string]any{+ "Query": query,"MaxLength": maxRawContentLength,"Results": docs,}diff --git a/backend/pkg/tools/firecrawl_test.go b/backend/pkg/tools/firecrawl_test.goindex 9c9a8313..0d1d02da 100644--- a/backend/pkg/tools/firecrawl_test.go+++ b/backend/pkg/tools/firecrawl_test.go@@ -329,7 +329,7 @@ func TestFirecrawlParseHTTPResponse_StatusAndDecodeErrors(t *testing.T) {StatusCode: tt.statusCode,Body: io.NopCloser(strings.NewReader(tt.body)),}- result, err := fc.parseHTTPResponse(t.Context(), resp)+ result, err := fc.parseHTTPResponse(t.Context(), "test query", resp)if !tt.wantErr {if err != nil {@@ -358,6 +358,10 @@ func TestFirecrawlBuildResult_WithSummarizer(t *testing.T) {if !strings.Contains(prompt, "<raw_content") {t.Fatalf("summarizer prompt must include raw content, got: %q", prompt)}+ // The user query must be rendered so the model can answer it.+ if !strings.Contains(prompt, `USER QUERY: "test query"`) {+ t.Fatalf("summarizer prompt must include the user query, got: %q", prompt)+ }// Source IDs must be one-based to line up with the numbered links.if !strings.Contains(prompt, `id="1"`) {t.Fatalf("summarizer prompt must use one-based source ids, got: %q", prompt)@@ -369,7 +373,7 @@ func TestFirecrawlBuildResult_WithSummarizer(t *testing.T) {},}- out := fc.buildFirecrawlResult(t.Context(), &firecrawlSearchResult{+ out := fc.buildFirecrawlResult(t.Context(), "test query", &firecrawlSearchResult{Success: true,Data: firecrawlData{Web: []firecrawlResult{@@ -394,7 +398,7 @@ func TestFirecrawlBuildResult_WithSummarizer(t *testing.T) {t.Run("falls back to markdown content when no summarizer", func(t *testing.T) {fc := &firecrawl{}- out := fc.buildFirecrawlResult(t.Context(), &firecrawlSearchResult{+ out := fc.buildFirecrawlResult(t.Context(), "test query", &firecrawlSearchResult{Success: true,Data: firecrawlData{Web: []firecrawlResult{@@ -419,7 +423,7 @@ func TestFirecrawlBuildResult_WithSummarizer(t *testing.T) {t.Run("no content sections when markdown is empty", func(t *testing.T) {fc := &firecrawl{}- out := fc.buildFirecrawlResult(t.Context(), &firecrawlSearchResult{+ out := fc.buildFirecrawlResult(t.Context(), "test query", &firecrawlSearchResult{Success: true,Data: firecrawlData{Web: []firecrawlResult{@@ -443,7 +447,7 @@ func TestFirecrawlBuildResult_WithSummarizer(t *testing.T) {t.Run("falls back to metadata url and title", func(t *testing.T) {fc := &firecrawl{}- out := fc.buildFirecrawlResult(t.Context(), &firecrawlSearchResult{+ out := fc.buildFirecrawlResult(t.Context(), "test query", &firecrawlSearchResult{Success: true,Data: firecrawlData{Web: []firecrawlResult{