High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: scrape --preview reports 'N page(s) discovered of 0' because totalDiscovered only ever counts sitemap URLs PIN-024
Luigi Agosti5ef2e6132363
diff --git a/docs/scrape.md b/docs/scrape.mdindex a63ea564..7b53a49d 100644--- a/docs/scrape.md+++ b/docs/scrape.md@@ -119,7 +119,7 @@ the verdict on each page but renders nothing.```schemaVersion, generatedAt, input-site: baseUrl, title, sitemapFound, totalDiscovered, sampledPages+site: baseUrl, title, sitemapFound, totalURLsInSitemap, sampledPagespageGroups[]: pattern, total, sampled, urls[] # site tree by URL patternpages[]:url, title, statusCode, contentTypediff --git a/internal/cli/actions/actions_scrape.go b/internal/cli/actions/actions_scrape.goindex f60c1233..23be1117 100644--- a/internal/cli/actions/actions_scrape.go+++ b/internal/cli/actions/actions_scrape.go@@ -131,8 +131,8 @@ func writeScrapeArtifacts(dir string, raw []byte, report scrape.Report, format s// routing verdict, and a snippet, so the caller can pick pages to expand with// `scrape <url> --only <picked-urls>`.func printPreviewSummary(report scrape.Report) {- fmt.Printf("Preview: %d page(s) discovered of %d · pick pages to expand with --only\n",- len(report.Pages), report.Site.TotalDiscovered)+ fmt.Printf("Preview: %s · pick pages to expand with --only\n",+ scrape.SampledPagesSummary(len(report.Pages), report.Site.TotalURLsInSitemap))for _, p := range report.Pages {verdict := ""if p.BrowserRecommended {diff --git a/internal/cli/actions/actions_scrape_test.go b/internal/cli/actions/actions_scrape_test.gonew file mode 100644index 00000000..5b24fac6--- /dev/null+++ b/internal/cli/actions/actions_scrape_test.go@@ -0,0 +1,71 @@+package actions++import (+ "strings"+ "testing"++ "github.com/pinchtab/pinchtab/internal/scrape"+)++func previewReport(totalURLsInSitemap int) scrape.Report {+ return scrape.Report{+ Site: scrape.SiteInfo{+ BaseURL: "https://example.com",+ TotalURLsInSitemap: totalURLsInSitemap,+ SampledPages: 2,+ },+ Pages: []scrape.Page{+ {URL: "https://example.com/", CharCount: 4229},+ {URL: "https://example.com/docs", CharCount: 2332},+ },+ }+}++func TestPrintPreviewSummaryWithoutSitemapOmitsTheTotal(t *testing.T) {+ report := previewReport(0)++ out := captureStdout(t, func() { printPreviewSummary(report) })++ header := strings.SplitN(out, "\n", 2)[0]+ if strings.Contains(header, " of ") {+ t.Fatalf("preview claims a total it does not have: %q", header)+ }+ if !strings.Contains(header, "2 page(s) sampled") {+ t.Fatalf("preview does not report the pages it lists: %q", header)+ }+ for _, page := range report.Pages {+ if !strings.Contains(out, page.URL) {+ t.Fatalf("preview omits %s:\n%s", page.URL, out)+ }+ }+}++func TestPrintPreviewSummaryWithSitemapNamesTheSitemapTotal(t *testing.T) {+ report := previewReport(42)++ out := captureStdout(t, func() { printPreviewSummary(report) })++ header := strings.SplitN(out, "\n", 2)[0]+ if !strings.Contains(header, "2 page(s) sampled of 42 in sitemap") {+ t.Fatalf("preview header = %q", header)+ }+}++// The preview line and the markdown report describe the same two numbers, so+// they must be built from the same summary rather than two hand-written forms.+func TestPreviewHeaderMatchesTheMarkdownReportLine(t *testing.T) {+ for _, total := range []int{0, 42} {+ report := previewReport(total)++ out := captureStdout(t, func() { printPreviewSummary(report) })+ header := strings.SplitN(out, "\n", 2)[0]+ summary := scrape.SampledPagesSummary(len(report.Pages), total)++ if !strings.Contains(header, summary) {+ t.Fatalf("total %d: preview header %q does not carry %q", total, header, summary)+ }+ if md := string(scrape.RenderMarkdown(report)); !strings.Contains(md, summary) {+ t.Fatalf("total %d: markdown report does not carry %q", total, summary)+ }+ }+}diff --git a/internal/scrape/render.go b/internal/scrape/render.goindex 3f24ba59..eb5171f6 100644--- a/internal/scrape/render.go+++ b/internal/scrape/render.go@@ -5,6 +5,16 @@ import ("strings")+// SampledPagesSummary describes how many pages a run sampled, naming the+// sitemap total only when a sitemap supplied one. Shared with the CLI preview+// so both surfaces use the same nouns for the same two numbers.+func SampledPagesSummary(sampled, totalURLsInSitemap int) string {+ if totalURLsInSitemap > 0 {+ return fmt.Sprintf("%d page(s) sampled of %d in sitemap", sampled, totalURLsInSitemap)+ }+ return fmt.Sprintf("%d page(s) sampled", sampled)+}+// RenderMarkdown renders the report as a single Markdown digest: site// overview, the page tree by URL pattern, then each page's content.func RenderMarkdown(r Report) []byte {@@ -15,7 +25,7 @@ func RenderMarkdown(r Report) []byte {}fmt.Fprintf(&b, "# Scrape Report — %s\n\n", title)fmt.Fprintf(&b, "- Base URL: %s\n", r.Site.BaseURL)- fmt.Fprintf(&b, "- Pages: %d sampled of %d discovered\n", r.Site.SampledPages, r.Site.TotalDiscovered)+ fmt.Fprintf(&b, "- Pages: %s\n", SampledPagesSummary(r.Site.SampledPages, r.Site.TotalURLsInSitemap))fmt.Fprintf(&b, "- Sources: %d http · %d browser-rendered · %d failed\n",r.Summary.HTTPPages, r.Summary.BrowserPages, r.Summary.FailedPages)fmt.Fprintf(&b, "- Generated: %s\n\n", r.GeneratedAt.Format("2006-01-02 15:04:05 UTC"))diff --git a/internal/scrape/run.go b/internal/scrape/run.goindex 9f3ef9ac..92c2d19d 100644--- a/internal/scrape/run.go+++ b/internal/scrape/run.go@@ -130,7 +130,7 @@ func URLListCrawler(urls []string, timeout time.Duration, guard CrawlGuard) Crawpages = append(pages, fetchOne(ctx, u, timeout, policy))}return &seaportal.ScrapeResult{- Site: seaportal.SiteInfo{BaseURL: originOf(clean[0]), TotalURLsInSitemap: len(clean)},+ Site: seaportal.SiteInfo{BaseURL: originOf(clean[0])},Pages: pages,}, nil}@@ -240,11 +240,11 @@ func Run(ctx context.Context, input Input, opts RunOptions, crawl Crawler, rendeGeneratedAt: time.Now().UTC(),Input: input,Site: SiteInfo{- BaseURL: res.Site.BaseURL,- Title: res.Site.Title,- SitemapFound: res.Site.SitemapFound,- TotalDiscovered: res.Site.TotalURLsInSitemap,- SampledPages: len(pages),+ BaseURL: res.Site.BaseURL,+ Title: res.Site.Title,+ SitemapFound: res.Site.SitemapFound,+ TotalURLsInSitemap: res.Site.TotalURLsInSitemap,+ SampledPages: len(pages),},PageGroups: fromSeaportalGroups(res.PageGroups),Pages: pages,diff --git a/internal/scrape/run_test.go b/internal/scrape/run_test.goindex 64071123..1cf0fd30 100644--- a/internal/scrape/run_test.go+++ b/internal/scrape/run_test.go@@ -362,3 +362,95 @@ func TestRunConcurrencyIsBounded(t *testing.T) {t.Fatalf("Run: %v", err)}}++// crawlWithoutSitemap is the shape every site without a usable sitemap+// produces: pages found by crawling, no sitemap total to report.+func crawlWithoutSitemap(pages ...seaportal.PageObject) Crawler {+ return func(context.Context) (*seaportal.ScrapeResult, error) {+ return &seaportal.ScrapeResult{+ Site: seaportal.SiteInfo{BaseURL: "https://example.com", SitemapFound: false},+ Pages: pages,+ }, nil+ }+}++func TestRunWithoutSitemapNeverReportsATotalBelowTheSampledPages(t *testing.T) {+ crawl := crawlWithoutSitemap(+ seaportal.PageObject{URL: "https://example.com/", Status: 200, Markdown: longMarkdown},+ seaportal.PageObject{URL: "https://example.com/docs", Status: 200, Markdown: longMarkdown},+ )++ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{Preview: true}, crawl, nil)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ if report.Site.TotalURLsInSitemap != 0 {+ t.Fatalf("TotalURLsInSitemap = %d, want 0 without a sitemap", report.Site.TotalURLsInSitemap)+ }+ summary := SampledPagesSummary(len(report.Pages), report.Site.TotalURLsInSitemap)+ if strings.Contains(summary, " of ") {+ t.Fatalf("summary %q claims a total the crawl never had", summary)+ }+ if !strings.Contains(summary, fmt.Sprintf("%d page(s) sampled", len(report.Pages))) {+ t.Fatalf("summary %q does not report the %d pages it lists", summary, len(report.Pages))+ }+ if md := string(RenderMarkdown(report)); !strings.Contains(md, summary) || strings.Contains(md, "of 0") {+ t.Fatalf("markdown report disagrees with the preview line: %q", firstLines(md, 4))+ }+}++func TestRunWithSitemapStillReportsTheSitemapTotal(t *testing.T) {+ pages := []seaportal.PageObject{+ {URL: "https://example.com/", Status: 200, Markdown: longMarkdown},+ {URL: "https://example.com/docs", Status: 200, Markdown: longMarkdown},+ }+ crawl := func(context.Context) (*seaportal.ScrapeResult, error) {+ return &seaportal.ScrapeResult{+ Site: seaportal.SiteInfo{BaseURL: "https://example.com", SitemapFound: true, TotalURLsInSitemap: 42},+ Pages: pages,+ }, nil+ }++ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{Preview: true}, crawl, nil)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ if !report.Site.SitemapFound || report.Site.TotalURLsInSitemap != 42 {+ t.Fatalf("sitemap info lost: found=%v total=%d", report.Site.SitemapFound, report.Site.TotalURLsInSitemap)+ }+ summary := SampledPagesSummary(len(report.Pages), report.Site.TotalURLsInSitemap)+ if !strings.Contains(summary, "42 in sitemap") {+ t.Fatalf("summary %q drops the sitemap total", summary)+ }+ if md := string(RenderMarkdown(report)); !strings.Contains(md, summary) {+ t.Fatalf("markdown report disagrees with the preview line: %q", firstLines(md, 4))+ }+}++func TestURLListCrawlerLeavesTheSitemapTotalUnset(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.Header().Set("Content-Type", "text/html; charset=utf-8")+ _, _ = fmt.Fprint(w, renderedHTML)+ }))+ defer srv.Close()+ guard := CrawlGuard{ValidateURL: func(string) error { return nil }}+ crawl := URLListCrawler([]string{srv.URL + "/a", srv.URL + "/b"}, 5*time.Second, guard)++ res, err := crawl(context.Background())+ if err != nil {+ t.Fatalf("crawl: %v", err)+ }+ if res.Site.TotalURLsInSitemap != 0 {+ t.Fatalf("TotalURLsInSitemap = %d, want 0: caller-supplied URLs never came from a sitemap", res.Site.TotalURLsInSitemap)+ }+}++func firstLines(s string, n int) string {+ lines := strings.Split(s, "\n")+ if len(lines) > n {+ lines = lines[:n]+ }+ return strings.Join(lines, "\n")+}diff --git a/internal/scrape/types.go b/internal/scrape/types.goindex 03055c71..b11fb442 100644--- a/internal/scrape/types.go+++ b/internal/scrape/types.go@@ -9,7 +9,7 @@ import "time"// SchemaVersion is the current scrape report schema version, embedded in// every report so consumers can detect format changes.-const SchemaVersion = "1.0"+const SchemaVersion = "2.0"// Page sources: how a page's content was obtained.const (@@ -73,11 +73,11 @@ type PageGroup struct {// SiteInfo describes the crawled site.type SiteInfo struct {- BaseURL string `json:"baseUrl"`- Title string `json:"title,omitempty"`- SitemapFound bool `json:"sitemapFound"`- TotalDiscovered int `json:"totalDiscovered"`- SampledPages int `json:"sampledPages"`+ BaseURL string `json:"baseUrl"`+ Title string `json:"title,omitempty"`+ SitemapFound bool `json:"sitemapFound"`+ TotalURLsInSitemap int `json:"totalURLsInSitemap,omitempty"`+ SampledPages int `json:"sampledPages"`}// Summary is the run roll-up.