Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
address review: bound & align firecrawl summarizer prompt, fix docs
rramprakash7ffa30ddc9a9
diff --git a/README.md b/README.mdindex f09cd31d..95d107d4 100644--- a/README.md+++ b/README.md@@ -3448,7 +3448,7 @@ go run cmd/ftester/main.go browser- **google**: Search the web using Google Custom Search- **duckduckgo**: Search the web using DuckDuckGo- **tavily**: Search using Tavily AI search engine-- **firecrawl**: Search using Firecrawl with full-page content scraping+- **firecrawl**: Search using Firecrawl with main-content markdown scraping- **traversaal**: Search using Traversaal AI search engine- **perplexity**: Search using Perplexity AI- **sploitus**: Search for security exploits, vulnerabilities (CVEs), and pentesting toolsdiff --git a/backend/docs/config.md b/backend/docs/config.mdindex 78b6b127..c7ab50ae 100644--- a/backend/docs/config.md+++ b/backend/docs/config.md@@ -1501,13 +1501,15 @@ tavilySearch: &functions.TavilySearchFunc{summarizer: cfg.Summarizer,},-// Firecrawl Search configuration-firecrawlSearch: &functions.FirecrawlSearchFunc{- apiKey: fte.cfg.FirecrawlAPIKey,- apiURL: fte.cfg.FirecrawlAPIURL,- proxyURL: fte.cfg.ProxyURL,- summarizer: cfg.Summarizer,-},+// Firecrawl Search configuration (FirecrawlAPIKey / FirecrawlAPIURL read from fte.cfg)+firecrawl := NewFirecrawlTool(+ fte.cfg,+ fte.flowID,+ cfg.TaskID,+ cfg.SubtaskID,+ fte.slp,+ cfg.Summarizer,+)// Perplexity Search configurationperplexitySearch: &functions.PerplexitySearchFunc{diff --git a/backend/docs/flow_execution.md b/backend/docs/flow_execution.mdindex 74f8e154..51f72c0b 100644--- a/backend/docs/flow_execution.md+++ b/backend/docs/flow_execution.md@@ -761,7 +761,7 @@ The Searcher Agent follows a strict hierarchy for information retrieval:3. **Priority 5: Deep Analysis Tools** - Complex research synthesis- `traversaal` - Structured answers for common questions- `tavily` - Research-grade exploration of technical topics- - `firecrawl` - Deep research with full-page content scraped from discovered sites+ - `firecrawl` - Deep research with main-content markdown scraped from discovered sites- `perplexity` - Comprehensive analysis with advanced reasoning**Available Search Engines**: Google, DuckDuckGo, Tavily, Firecrawl, Traversaal, Perplexity, Sploitus, Searxng@@ -770,7 +770,7 @@ The Searcher Agent follows a strict hierarchy for information retrieval:- **Google** - Custom Search API with CX key and language restrictions- **DuckDuckGo** - Anonymous search with VQD token authentication- **Tavily** - Advanced research with raw content and citations-- **Firecrawl** - Web search with full-page markdown content scraped from each result+- **Firecrawl** - Web search with main-content markdown scraped from each result- **Perplexity** - AI-powered synthesis with configurable context size- **Traversaal** - Structured Q&A responses with web links- **Sploitus** - Search for security exploits and pentest toolsdiff --git a/backend/pkg/templates/prompts/searcher.tmpl b/backend/pkg/templates/prompts/searcher.tmplindex 153537fb..81c29637 100644--- a/backend/pkg/templates/prompts/searcher.tmpl+++ b/backend/pkg/templates/prompts/searcher.tmpl@@ -106,7 +106,7 @@ External search engines and the answer vector store are indexed in English and s<deep_analysis_tools><tool name="tavily" priority="5">For research-grade exploration of complex technical topics</tool>-<tool name="firecrawl" priority="5">For deep research with full-page content scraped directly from the discovered web sites</tool>+<tool name="firecrawl" priority="5">For deep research with main-content markdown extracted directly from the discovered web sites</tool><tool name="perplexity" priority="5">For comprehensive analysis with advanced reasoning</tool><tool name="traversaal" priority="4">For discovering structured answers to common questions</tool></deep_analysis_tools>diff --git a/backend/pkg/tools/firecrawl.go b/backend/pkg/tools/firecrawl.goindex 5d41695e..7a4f9cc5 100644--- a/backend/pkg/tools/firecrawl.go+++ b/backend/pkg/tools/firecrawl.go@@ -294,6 +294,17 @@ func (f *firecrawl) getContentFromResults(results []firecrawlResult) string {return writer.String()}+// firecrawlPromptDoc is a single source record fed to the summarizer. IDs are+// one-based so they line up with the numbered links in the tool output, and the+// title/URL use the metadata fallbacks so nothing renders empty. Markdown is+// bounded per document so an oversized or hostile page can't blow up the prompt.+type firecrawlPromptDoc struct {+ ID int+ Title string+ URL string+ Markdown string+}+func (f *firecrawl) getSummarizePrompt(result *firecrawlSearchResult) (string, error) {templateText := `<instructions>TASK: Summarize web search results for the following user query:@@ -321,17 +332,31 @@ FORMAT:The summary MUST provide complete answers to the user's query, preserving all relevant information.</instructions>-{{range $index, $result := .Results}}-{{if $result.Markdown}}-<raw_content id="{{$index}}" title="{{$result.Title}}" url="{{$result.URL}}">-{{$result.Markdown}}+{{range .Results}}+<raw_content id="{{.ID}}" title="{{.Title}}" url="{{.URL}}">+{{.Markdown}}</raw_content>-{{end}}{{end}}`+ // Keep source IDs aligned with the one-based numbering of the links section,+ // so a result without markdown still leaves a gap rather than renumbering.+ docs := make([]firecrawlPromptDoc, 0, len(result.Data.Web))+ for i, res := range result.Data.Web {+ if res.Markdown == "" {+ continue+ }+ markdown := res.Markdown[:min(len(res.Markdown), maxRawContentLength)]+ docs = append(docs, firecrawlPromptDoc{+ ID: i + 1,+ Title: res.resolvedTitle(),+ URL: res.resolvedURL(),+ Markdown: markdown,+ })+ }+templateContext := map[string]any{"MaxLength": maxRawContentLength,- "Results": result.Data.Web,+ "Results": docs,}tmpl, err := template.New("summarize").Parse(templateText)diff --git a/backend/pkg/tools/firecrawl_test.go b/backend/pkg/tools/firecrawl_test.goindex cb2f96d3..9c9a8313 100644--- a/backend/pkg/tools/firecrawl_test.go+++ b/backend/pkg/tools/firecrawl_test.go@@ -358,6 +358,13 @@ func TestFirecrawlBuildResult_WithSummarizer(t *testing.T) {if !strings.Contains(prompt, "<raw_content") {t.Fatalf("summarizer prompt must include raw content, got: %q", prompt)}+ // Source IDs must be one-based to line up with the numbered links.+ if !strings.Contains(prompt, `id="1"`) {+ t.Fatalf("summarizer prompt must use one-based source ids, got: %q", prompt)+ }+ if !strings.Contains(prompt, `title="Title"`) || !strings.Contains(prompt, `url="https://example.com"`) {+ t.Fatalf("summarizer prompt must carry resolved title/url, got: %q", prompt)+ }return "short summary", nil},}diff --git a/backend/pkg/tools/registry.go b/backend/pkg/tools/registry.goindex bd21a413..877c58e4 100644--- a/backend/pkg/tools/registry.go+++ b/backend/pkg/tools/registry.go@@ -236,7 +236,7 @@ var registryDefinitions = map[string]llms.FunctionDefinition{FirecrawlToolName: {Name: FirecrawlToolName,Description: "Search in the firecrawl search engine, it combines web search with page scraping to return " +- "full-page markdown content for each result, ideal for deep research on complex technical topics " ++ "the main-content markdown for each result, ideal for deep research on complex technical topics " +"and reading documentation directly from the discovered web sites",Parameters: reflector.Reflect(&SearchAction{}),},