High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat: expose scrape as the pinchtab_scrape mcp tool
Luigi Agosti1aa390e8ccec
diff --git a/internal/mcp/handlers.go b/internal/mcp/handlers.goindex a284dba0..6930c09e 100644--- a/internal/mcp/handlers.go+++ b/internal/mcp/handlers.go@@ -61,5 +61,7 @@ func handlerMap(c *Client) map[string]func(context.Context, mcp.CallToolRequest)"pinchtab_record_status": handleRecordStatus(c),"pinchtab_dialog": handleDialog(c),++ "pinchtab_scrape": handleScrape(c),}}diff --git a/internal/mcp/handlers_scrape.go b/internal/mcp/handlers_scrape.gonew file mode 100644index 00000000..39acc782--- /dev/null+++ b/internal/mcp/handlers_scrape.go@@ -0,0 +1,81 @@+package mcp++import (+ "context"+ "strings"+ "time"++ "github.com/mark3labs/mcp-go/mcp"+)++// scrapeMCPTimeout bounds a scrape driven over MCP: the HTTP crawl plus any+// browser-rendered pages. Matches the CLI/HTTP client-side scrape budget, well+// above the MCP client's default per-request timeout.+const scrapeMCPTimeout = 15 * time.Minute++func handleScrape(c *Client) func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) {+ return func(ctx context.Context, r mcp.CallToolRequest) (*mcp.CallToolResult, error) {+ u, err := r.RequireString("url")+ if err != nil {+ return mcp.NewToolResultError(err.Error()), nil+ }+ payload := map[string]any{"url": u}+ if v, ok := optInt(r, "maxPages"); ok {+ payload["maxPages"] = v+ }+ if v, ok := optInt(r, "maxPerPattern"); ok {+ payload["maxPerPattern"] = v+ }+ if v, ok := optInt(r, "concurrency"); ok {+ payload["concurrency"] = v+ }+ if v, ok := optInt(r, "timeoutSeconds"); ok {+ payload["timeoutSeconds"] = v+ }+ if v, ok := optBool(r, "preview"); ok && v {+ payload["preview"] = true+ }+ if v, ok := optBool(r, "enrichAll"); ok && v {+ payload["enrichAll"] = true+ }+ if v, ok := optBool(r, "noBrowser"); ok && v {+ payload["noBrowser"] = true+ }+ if list := splitCommaList(optString(r, "only")); len(list) > 0 {+ payload["only"] = list+ }+ if list := splitCommaList(optString(r, "include")); len(list) > 0 {+ payload["includePatterns"] = list+ }+ if list := splitCommaList(optString(r, "exclude")); len(list) > 0 {+ payload["excludePatterns"] = list+ }+ if browser := optString(r, "browser"); browser != "" {+ payload["browser"] = browser+ }+ // A multi-page scrape legitimately runs for minutes; the default MCP+ // client timeout is far too short for it.+ body, code, err := c.withTimeout(scrapeMCPTimeout).Post(ctx, "/scrape", payload)+ if err != nil {+ return mcp.NewToolResultError(err.Error()), nil+ }+ return resultFromBytes(body, code)+ }+}++// splitCommaList splits a comma- or newline-separated string into trimmed,+// non-empty items — the MCP-friendly way to pass the scrape URL/pattern lists+// without an array schema.+func splitCommaList(s string) []string {+ if strings.TrimSpace(s) == "" {+ return nil+ }+ fields := strings.FieldsFunc(s, func(r rune) bool { return r == ',' || r == '\n' })+ out := make([]string, 0, len(fields))+ for _, f := range fields {+ if f = strings.TrimSpace(f); f != "" {+ out = append(out, f)+ }+ }+ return out+}diff --git a/internal/mcp/handlers_scrape_test.go b/internal/mcp/handlers_scrape_test.gonew file mode 100644index 00000000..c2fa97e5--- /dev/null+++ b/internal/mcp/handlers_scrape_test.go@@ -0,0 +1,50 @@+package mcp++import (+ "strings"+ "testing"+)++func TestHandleScrape(t *testing.T) {+ srv := mockPinchTab()+ defer srv.Close()++ r := callTool(t, "pinchtab_scrape", map[string]any{+ "url": "https://example.com",+ "preview": true,+ }, srv)++ text := resultText(t, r)+ if !strings.Contains(text, "/scrape") {+ t.Errorf("expected /scrape path, got %s", text)+ }+ if !strings.Contains(text, `"preview":true`) {+ t.Errorf("preview flag not forwarded: %s", text)+ }+}++func TestHandleScrapeSplitsCommaLists(t *testing.T) {+ srv := mockPinchTab()+ defer srv.Close()++ r := callTool(t, "pinchtab_scrape", map[string]any{+ "url": "https://example.com",+ "only": "https://example.com/a, https://example.com/b",+ }, srv)++ // The comma-separated string must reach the server as a JSON array of URLs.+ text := resultText(t, r)+ if !strings.Contains(text, "https://example.com/a") || !strings.Contains(text, "https://example.com/b") {+ t.Errorf("only list not split/forwarded: %s", text)+ }+}++func TestHandleScrapeMissingURL(t *testing.T) {+ srv := mockPinchTab()+ defer srv.Close()++ r := callTool(t, "pinchtab_scrape", map[string]any{}, srv)+ if !r.IsError {+ t.Error("expected error for missing url")+ }+}diff --git a/internal/mcp/server_test.go b/internal/mcp/server_test.goindex 116f2b69..ea84dad8 100644--- a/internal/mcp/server_test.go+++ b/internal/mcp/server_test.go@@ -18,8 +18,8 @@ func TestNewServerRegistersAllTools(t *testing.T) {// The server should have registered all tools.// We verify by checking that NewServer doesn't panic — the panic// in NewServer fires if any tool lacks a handler.- if len(tools) != 42 {- t.Errorf("expected 42 tools, got %d", len(tools))+ if len(tools) != 43 {+ t.Errorf("expected 43 tools, got %d", len(tools))}}diff --git a/internal/mcp/tools.go b/internal/mcp/tools.goindex e6174d4b..afc58179 100644--- a/internal/mcp/tools.go+++ b/internal/mcp/tools.go@@ -344,5 +344,22 @@ func allTools() []mcp.Tool {mcp.WithString("text", mcp.Description("Text to enter for prompt dialogs (only used with accept)")),mcp.WithString("tabId", mcp.Description("Target tab ID")),),++ mcp.NewTool("pinchtab_scrape",+ mcp.WithDescription("Scrape a whole site into a page tree of markdown. Pages are discovered and extracted over plain HTTP first (SeaPortal: sitemap/link crawl, URL-pattern sampling); only pages whose HTTP extraction is thin, blocked, or failed are re-rendered in the real browser, so JS-only content still lands. Each page records its content source (http|browser) and routing verdict. For a LARGE site, first call with preview=true for a cheap outline (titles, sizes, snippets, routing verdicts — no bodies, no browser), then expand the pages you want with only=<comma-separated urls>. Full reports can be large, so prefer preview then only over scraping everything."),+ mcp.WithString("url", mcp.Required(), mcp.Description("Site URL to crawl (discovery seeds from this host's root)")),+ mcp.WithBoolean("preview", mcp.Description("Outline only: page tree, sizes (charCount), and snippets — no browser rendering or full bodies. Survey a large site before expanding.")),+ mcp.WithString("only", mcp.Description("Comma-separated URLs to expand at full fidelity instead of crawling — the drill-down after a preview")),+ mcp.WithNumber("maxPages", mcp.Description("Maximum pages sampled across the site (default 50)")),+ mcp.WithNumber("maxPerPattern", mcp.Description("Maximum pages sampled per URL-pattern group (default 8)")),+ mcp.WithString("include", mcp.Description("Comma-separated regexes; only crawl URLs matching one")),+ mcp.WithString("exclude", mcp.Description("Comma-separated regexes; skip URLs matching one")),+ mcp.WithNumber("concurrency", mcp.Description("Pages browser-rendered in parallel (default 2, max 8)")),+ mcp.WithBoolean("enrichAll", mcp.Description("Browser-render every reachable page, ignoring routing")),+ mcp.WithBoolean("noBrowser", mcp.Description("HTTP crawl only; record routing verdicts without browser rendering")),+ mcp.WithNumber("timeoutSeconds", mcp.Description("Overall HTTP crawl timeout in seconds (default 60)")),+ mcp.WithString("browser",+ mcp.Description("Browser to use for enrichment (e.g. chrome, cloak, ghost-chrome).")),+ ),}}