Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

feat: expose scrape as the pinchtab_scrape mcp tool

Luigi Agosti1aa390e8ccec
diff --git a/internal/mcp/handlers.go b/internal/mcp/handlers.go
index a284dba0..6930c09e 100644
--- a/internal/mcp/handlers.go
+++ b/internal/mcp/handlers.go
@@ -61,5 +61,7 @@ func handlerMap(c *Client) map[string]func(context.Context, mcp.CallToolRequest)
"pinchtab_record_status": handleRecordStatus(c),
"pinchtab_dialog": handleDialog(c),
+
+ "pinchtab_scrape": handleScrape(c),
}
}
diff --git a/internal/mcp/handlers_scrape.go b/internal/mcp/handlers_scrape.go
new file mode 100644
index 00000000..39acc782
--- /dev/null
+++ b/internal/mcp/handlers_scrape.go
@@ -0,0 +1,81 @@
+package mcp
+
+import (
+ "context"
+ "strings"
+ "time"
+
+ "github.com/mark3labs/mcp-go/mcp"
+)
+
+// scrapeMCPTimeout bounds a scrape driven over MCP: the HTTP crawl plus any
+// browser-rendered pages. Matches the CLI/HTTP client-side scrape budget, well
+// above the MCP client's default per-request timeout.
+const scrapeMCPTimeout = 15 * time.Minute
+
+func handleScrape(c *Client) func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) {
+ return func(ctx context.Context, r mcp.CallToolRequest) (*mcp.CallToolResult, error) {
+ u, err := r.RequireString("url")
+ if err != nil {
+ return mcp.NewToolResultError(err.Error()), nil
+ }
+ payload := map[string]any{"url": u}
+ if v, ok := optInt(r, "maxPages"); ok {
+ payload["maxPages"] = v
+ }
+ if v, ok := optInt(r, "maxPerPattern"); ok {
+ payload["maxPerPattern"] = v
+ }
+ if v, ok := optInt(r, "concurrency"); ok {
+ payload["concurrency"] = v
+ }
+ if v, ok := optInt(r, "timeoutSeconds"); ok {
+ payload["timeoutSeconds"] = v
+ }
+ if v, ok := optBool(r, "preview"); ok && v {
+ payload["preview"] = true
+ }
+ if v, ok := optBool(r, "enrichAll"); ok && v {
+ payload["enrichAll"] = true
+ }
+ if v, ok := optBool(r, "noBrowser"); ok && v {
+ payload["noBrowser"] = true
+ }
+ if list := splitCommaList(optString(r, "only")); len(list) > 0 {
+ payload["only"] = list
+ }
+ if list := splitCommaList(optString(r, "include")); len(list) > 0 {
+ payload["includePatterns"] = list
+ }
+ if list := splitCommaList(optString(r, "exclude")); len(list) > 0 {
+ payload["excludePatterns"] = list
+ }
+ if browser := optString(r, "browser"); browser != "" {
+ payload["browser"] = browser
+ }
+ // A multi-page scrape legitimately runs for minutes; the default MCP
+ // client timeout is far too short for it.
+ body, code, err := c.withTimeout(scrapeMCPTimeout).Post(ctx, "/scrape", payload)
+ if err != nil {
+ return mcp.NewToolResultError(err.Error()), nil
+ }
+ return resultFromBytes(body, code)
+ }
+}
+
+// splitCommaList splits a comma- or newline-separated string into trimmed,
+// non-empty items — the MCP-friendly way to pass the scrape URL/pattern lists
+// without an array schema.
+func splitCommaList(s string) []string {
+ if strings.TrimSpace(s) == "" {
+ return nil
+ }
+ fields := strings.FieldsFunc(s, func(r rune) bool { return r == ',' || r == '\n' })
+ out := make([]string, 0, len(fields))
+ for _, f := range fields {
+ if f = strings.TrimSpace(f); f != "" {
+ out = append(out, f)
+ }
+ }
+ return out
+}
diff --git a/internal/mcp/handlers_scrape_test.go b/internal/mcp/handlers_scrape_test.go
new file mode 100644
index 00000000..c2fa97e5
--- /dev/null
+++ b/internal/mcp/handlers_scrape_test.go
@@ -0,0 +1,50 @@
+package mcp
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestHandleScrape(t *testing.T) {
+ srv := mockPinchTab()
+ defer srv.Close()
+
+ r := callTool(t, "pinchtab_scrape", map[string]any{
+ "url": "https://example.com",
+ "preview": true,
+ }, srv)
+
+ text := resultText(t, r)
+ if !strings.Contains(text, "/scrape") {
+ t.Errorf("expected /scrape path, got %s", text)
+ }
+ if !strings.Contains(text, `"preview":true`) {
+ t.Errorf("preview flag not forwarded: %s", text)
+ }
+}
+
+func TestHandleScrapeSplitsCommaLists(t *testing.T) {
+ srv := mockPinchTab()
+ defer srv.Close()
+
+ r := callTool(t, "pinchtab_scrape", map[string]any{
+ "url": "https://example.com",
+ "only": "https://example.com/a, https://example.com/b",
+ }, srv)
+
+ // The comma-separated string must reach the server as a JSON array of URLs.
+ text := resultText(t, r)
+ if !strings.Contains(text, "https://example.com/a") || !strings.Contains(text, "https://example.com/b") {
+ t.Errorf("only list not split/forwarded: %s", text)
+ }
+}
+
+func TestHandleScrapeMissingURL(t *testing.T) {
+ srv := mockPinchTab()
+ defer srv.Close()
+
+ r := callTool(t, "pinchtab_scrape", map[string]any{}, srv)
+ if !r.IsError {
+ t.Error("expected error for missing url")
+ }
+}
diff --git a/internal/mcp/server_test.go b/internal/mcp/server_test.go
index 116f2b69..ea84dad8 100644
--- a/internal/mcp/server_test.go
+++ b/internal/mcp/server_test.go
@@ -18,8 +18,8 @@ func TestNewServerRegistersAllTools(t *testing.T) {
// The server should have registered all tools.
// We verify by checking that NewServer doesn't panic — the panic
// in NewServer fires if any tool lacks a handler.
- if len(tools) != 42 {
- t.Errorf("expected 42 tools, got %d", len(tools))
+ if len(tools) != 43 {
+ t.Errorf("expected 43 tools, got %d", len(tools))
}
}
diff --git a/internal/mcp/tools.go b/internal/mcp/tools.go
index e6174d4b..afc58179 100644
--- a/internal/mcp/tools.go
+++ b/internal/mcp/tools.go
@@ -344,5 +344,22 @@ func allTools() []mcp.Tool {
mcp.WithString("text", mcp.Description("Text to enter for prompt dialogs (only used with accept)")),
mcp.WithString("tabId", mcp.Description("Target tab ID")),
),
+
+ mcp.NewTool("pinchtab_scrape",
+ mcp.WithDescription("Scrape a whole site into a page tree of markdown. Pages are discovered and extracted over plain HTTP first (SeaPortal: sitemap/link crawl, URL-pattern sampling); only pages whose HTTP extraction is thin, blocked, or failed are re-rendered in the real browser, so JS-only content still lands. Each page records its content source (http|browser) and routing verdict. For a LARGE site, first call with preview=true for a cheap outline (titles, sizes, snippets, routing verdicts — no bodies, no browser), then expand the pages you want with only=<comma-separated urls>. Full reports can be large, so prefer preview then only over scraping everything."),
+ mcp.WithString("url", mcp.Required(), mcp.Description("Site URL to crawl (discovery seeds from this host's root)")),
+ mcp.WithBoolean("preview", mcp.Description("Outline only: page tree, sizes (charCount), and snippets — no browser rendering or full bodies. Survey a large site before expanding.")),
+ mcp.WithString("only", mcp.Description("Comma-separated URLs to expand at full fidelity instead of crawling — the drill-down after a preview")),
+ mcp.WithNumber("maxPages", mcp.Description("Maximum pages sampled across the site (default 50)")),
+ mcp.WithNumber("maxPerPattern", mcp.Description("Maximum pages sampled per URL-pattern group (default 8)")),
+ mcp.WithString("include", mcp.Description("Comma-separated regexes; only crawl URLs matching one")),
+ mcp.WithString("exclude", mcp.Description("Comma-separated regexes; skip URLs matching one")),
+ mcp.WithNumber("concurrency", mcp.Description("Pages browser-rendered in parallel (default 2, max 8)")),
+ mcp.WithBoolean("enrichAll", mcp.Description("Browser-render every reachable page, ignoring routing")),
+ mcp.WithBoolean("noBrowser", mcp.Description("HTTP crawl only; record routing verdicts without browser rendering")),
+ mcp.WithNumber("timeoutSeconds", mcp.Description("Overall HTTP crawl timeout in seconds (default 60)")),
+ mcp.WithString("browser",
+ mcp.Description("Browser to use for enrichment (e.g. chrome, cloak, ghost-chrome).")),
+ ),
}
}