Skip to content

Fully autonomous AI Agents system capable of performing complex penetration testing tasks

address review: bound & align firecrawl summarizer prompt, fix docs

rramprakash7ffa30ddc9a9
diff --git a/README.md b/README.md
index f09cd31d..95d107d4 100644
--- a/README.md
+++ b/README.md
@@ -3448,7 +3448,7 @@ go run cmd/ftester/main.go browser
- **google**: Search the web using Google Custom Search
- **duckduckgo**: Search the web using DuckDuckGo
- **tavily**: Search using Tavily AI search engine
-- **firecrawl**: Search using Firecrawl with full-page content scraping
+- **firecrawl**: Search using Firecrawl with main-content markdown scraping
- **traversaal**: Search using Traversaal AI search engine
- **perplexity**: Search using Perplexity AI
- **sploitus**: Search for security exploits, vulnerabilities (CVEs), and pentesting tools
diff --git a/backend/docs/config.md b/backend/docs/config.md
index 78b6b127..c7ab50ae 100644
--- a/backend/docs/config.md
+++ b/backend/docs/config.md
@@ -1501,13 +1501,15 @@ tavilySearch: &functions.TavilySearchFunc{
summarizer: cfg.Summarizer,
},
-// Firecrawl Search configuration
-firecrawlSearch: &functions.FirecrawlSearchFunc{
- apiKey: fte.cfg.FirecrawlAPIKey,
- apiURL: fte.cfg.FirecrawlAPIURL,
- proxyURL: fte.cfg.ProxyURL,
- summarizer: cfg.Summarizer,
-},
+// Firecrawl Search configuration (FirecrawlAPIKey / FirecrawlAPIURL read from fte.cfg)
+firecrawl := NewFirecrawlTool(
+ fte.cfg,
+ fte.flowID,
+ cfg.TaskID,
+ cfg.SubtaskID,
+ fte.slp,
+ cfg.Summarizer,
+)
// Perplexity Search configuration
perplexitySearch: &functions.PerplexitySearchFunc{
diff --git a/backend/docs/flow_execution.md b/backend/docs/flow_execution.md
index 74f8e154..51f72c0b 100644
--- a/backend/docs/flow_execution.md
+++ b/backend/docs/flow_execution.md
@@ -761,7 +761,7 @@ The Searcher Agent follows a strict hierarchy for information retrieval:
3. **Priority 5: Deep Analysis Tools** - Complex research synthesis
- `traversaal` - Structured answers for common questions
- `tavily` - Research-grade exploration of technical topics
- - `firecrawl` - Deep research with full-page content scraped from discovered sites
+ - `firecrawl` - Deep research with main-content markdown scraped from discovered sites
- `perplexity` - Comprehensive analysis with advanced reasoning
**Available Search Engines**: Google, DuckDuckGo, Tavily, Firecrawl, Traversaal, Perplexity, Sploitus, Searxng
@@ -770,7 +770,7 @@ The Searcher Agent follows a strict hierarchy for information retrieval:
- **Google** - Custom Search API with CX key and language restrictions
- **DuckDuckGo** - Anonymous search with VQD token authentication
- **Tavily** - Advanced research with raw content and citations
-- **Firecrawl** - Web search with full-page markdown content scraped from each result
+- **Firecrawl** - Web search with main-content markdown scraped from each result
- **Perplexity** - AI-powered synthesis with configurable context size
- **Traversaal** - Structured Q&A responses with web links
- **Sploitus** - Search for security exploits and pentest tools
diff --git a/backend/pkg/templates/prompts/searcher.tmpl b/backend/pkg/templates/prompts/searcher.tmpl
index 153537fb..81c29637 100644
--- a/backend/pkg/templates/prompts/searcher.tmpl
+++ b/backend/pkg/templates/prompts/searcher.tmpl
@@ -106,7 +106,7 @@ External search engines and the answer vector store are indexed in English and s
<deep_analysis_tools>
<tool name="tavily" priority="5">For research-grade exploration of complex technical topics</tool>
-<tool name="firecrawl" priority="5">For deep research with full-page content scraped directly from the discovered web sites</tool>
+<tool name="firecrawl" priority="5">For deep research with main-content markdown extracted directly from the discovered web sites</tool>
<tool name="perplexity" priority="5">For comprehensive analysis with advanced reasoning</tool>
<tool name="traversaal" priority="4">For discovering structured answers to common questions</tool>
</deep_analysis_tools>
diff --git a/backend/pkg/tools/firecrawl.go b/backend/pkg/tools/firecrawl.go
index 5d41695e..7a4f9cc5 100644
--- a/backend/pkg/tools/firecrawl.go
+++ b/backend/pkg/tools/firecrawl.go
@@ -294,6 +294,17 @@ func (f *firecrawl) getContentFromResults(results []firecrawlResult) string {
return writer.String()
}
+// firecrawlPromptDoc is a single source record fed to the summarizer. IDs are
+// one-based so they line up with the numbered links in the tool output, and the
+// title/URL use the metadata fallbacks so nothing renders empty. Markdown is
+// bounded per document so an oversized or hostile page can't blow up the prompt.
+type firecrawlPromptDoc struct {
+ ID int
+ Title string
+ URL string
+ Markdown string
+}
+
func (f *firecrawl) getSummarizePrompt(result *firecrawlSearchResult) (string, error) {
templateText := `<instructions>
TASK: Summarize web search results for the following user query:
@@ -321,17 +332,31 @@ FORMAT:
The summary MUST provide complete answers to the user's query, preserving all relevant information.
</instructions>
-{{range $index, $result := .Results}}
-{{if $result.Markdown}}
-<raw_content id="{{$index}}" title="{{$result.Title}}" url="{{$result.URL}}">
-{{$result.Markdown}}
+{{range .Results}}
+<raw_content id="{{.ID}}" title="{{.Title}}" url="{{.URL}}">
+{{.Markdown}}
</raw_content>
-{{end}}
{{end}}`
+ // Keep source IDs aligned with the one-based numbering of the links section,
+ // so a result without markdown still leaves a gap rather than renumbering.
+ docs := make([]firecrawlPromptDoc, 0, len(result.Data.Web))
+ for i, res := range result.Data.Web {
+ if res.Markdown == "" {
+ continue
+ }
+ markdown := res.Markdown[:min(len(res.Markdown), maxRawContentLength)]
+ docs = append(docs, firecrawlPromptDoc{
+ ID: i + 1,
+ Title: res.resolvedTitle(),
+ URL: res.resolvedURL(),
+ Markdown: markdown,
+ })
+ }
+
templateContext := map[string]any{
"MaxLength": maxRawContentLength,
- "Results": result.Data.Web,
+ "Results": docs,
}
tmpl, err := template.New("summarize").Parse(templateText)
diff --git a/backend/pkg/tools/firecrawl_test.go b/backend/pkg/tools/firecrawl_test.go
index cb2f96d3..9c9a8313 100644
--- a/backend/pkg/tools/firecrawl_test.go
+++ b/backend/pkg/tools/firecrawl_test.go
@@ -358,6 +358,13 @@ func TestFirecrawlBuildResult_WithSummarizer(t *testing.T) {
if !strings.Contains(prompt, "<raw_content") {
t.Fatalf("summarizer prompt must include raw content, got: %q", prompt)
}
+ // Source IDs must be one-based to line up with the numbered links.
+ if !strings.Contains(prompt, `id="1"`) {
+ t.Fatalf("summarizer prompt must use one-based source ids, got: %q", prompt)
+ }
+ if !strings.Contains(prompt, `title="Title"`) || !strings.Contains(prompt, `url="https://example.com"`) {
+ t.Fatalf("summarizer prompt must carry resolved title/url, got: %q", prompt)
+ }
return "short summary", nil
},
}
diff --git a/backend/pkg/tools/registry.go b/backend/pkg/tools/registry.go
index bd21a413..877c58e4 100644
--- a/backend/pkg/tools/registry.go
+++ b/backend/pkg/tools/registry.go
@@ -236,7 +236,7 @@ var registryDefinitions = map[string]llms.FunctionDefinition{
FirecrawlToolName: {
Name: FirecrawlToolName,
Description: "Search in the firecrawl search engine, it combines web search with page scraping to return " +
- "full-page markdown content for each result, ideal for deep research on complex technical topics " +
+ "the main-content markdown for each result, ideal for deep research on complex technical topics " +
"and reading documentation directly from the discovered web sites",
Parameters: reflector.Reflect(&SearchAction{}),
},