Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: scrape --preview reports 'N page(s) discovered of 0' because totalDiscovered only ever counts sitemap URLs PIN-024

Luigi Agosti5ef2e6132363
diff --git a/docs/scrape.md b/docs/scrape.md
index a63ea564..7b53a49d 100644
--- a/docs/scrape.md
+++ b/docs/scrape.md
@@ -119,7 +119,7 @@ the verdict on each page but renders nothing.
```
schemaVersion, generatedAt, input
-site: baseUrl, title, sitemapFound, totalDiscovered, sampledPages
+site: baseUrl, title, sitemapFound, totalURLsInSitemap, sampledPages
pageGroups[]: pattern, total, sampled, urls[] # site tree by URL pattern
pages[]:
url, title, statusCode, contentType
diff --git a/internal/cli/actions/actions_scrape.go b/internal/cli/actions/actions_scrape.go
index f60c1233..23be1117 100644
--- a/internal/cli/actions/actions_scrape.go
+++ b/internal/cli/actions/actions_scrape.go
@@ -131,8 +131,8 @@ func writeScrapeArtifacts(dir string, raw []byte, report scrape.Report, format s
// routing verdict, and a snippet, so the caller can pick pages to expand with
// `scrape <url> --only <picked-urls>`.
func printPreviewSummary(report scrape.Report) {
- fmt.Printf("Preview: %d page(s) discovered of %d · pick pages to expand with --only\n",
- len(report.Pages), report.Site.TotalDiscovered)
+ fmt.Printf("Preview: %s · pick pages to expand with --only\n",
+ scrape.SampledPagesSummary(len(report.Pages), report.Site.TotalURLsInSitemap))
for _, p := range report.Pages {
verdict := ""
if p.BrowserRecommended {
diff --git a/internal/cli/actions/actions_scrape_test.go b/internal/cli/actions/actions_scrape_test.go
new file mode 100644
index 00000000..5b24fac6
--- /dev/null
+++ b/internal/cli/actions/actions_scrape_test.go
@@ -0,0 +1,71 @@
+package actions
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/pinchtab/pinchtab/internal/scrape"
+)
+
+func previewReport(totalURLsInSitemap int) scrape.Report {
+ return scrape.Report{
+ Site: scrape.SiteInfo{
+ BaseURL: "https://example.com",
+ TotalURLsInSitemap: totalURLsInSitemap,
+ SampledPages: 2,
+ },
+ Pages: []scrape.Page{
+ {URL: "https://example.com/", CharCount: 4229},
+ {URL: "https://example.com/docs", CharCount: 2332},
+ },
+ }
+}
+
+func TestPrintPreviewSummaryWithoutSitemapOmitsTheTotal(t *testing.T) {
+ report := previewReport(0)
+
+ out := captureStdout(t, func() { printPreviewSummary(report) })
+
+ header := strings.SplitN(out, "\n", 2)[0]
+ if strings.Contains(header, " of ") {
+ t.Fatalf("preview claims a total it does not have: %q", header)
+ }
+ if !strings.Contains(header, "2 page(s) sampled") {
+ t.Fatalf("preview does not report the pages it lists: %q", header)
+ }
+ for _, page := range report.Pages {
+ if !strings.Contains(out, page.URL) {
+ t.Fatalf("preview omits %s:\n%s", page.URL, out)
+ }
+ }
+}
+
+func TestPrintPreviewSummaryWithSitemapNamesTheSitemapTotal(t *testing.T) {
+ report := previewReport(42)
+
+ out := captureStdout(t, func() { printPreviewSummary(report) })
+
+ header := strings.SplitN(out, "\n", 2)[0]
+ if !strings.Contains(header, "2 page(s) sampled of 42 in sitemap") {
+ t.Fatalf("preview header = %q", header)
+ }
+}
+
+// The preview line and the markdown report describe the same two numbers, so
+// they must be built from the same summary rather than two hand-written forms.
+func TestPreviewHeaderMatchesTheMarkdownReportLine(t *testing.T) {
+ for _, total := range []int{0, 42} {
+ report := previewReport(total)
+
+ out := captureStdout(t, func() { printPreviewSummary(report) })
+ header := strings.SplitN(out, "\n", 2)[0]
+ summary := scrape.SampledPagesSummary(len(report.Pages), total)
+
+ if !strings.Contains(header, summary) {
+ t.Fatalf("total %d: preview header %q does not carry %q", total, header, summary)
+ }
+ if md := string(scrape.RenderMarkdown(report)); !strings.Contains(md, summary) {
+ t.Fatalf("total %d: markdown report does not carry %q", total, summary)
+ }
+ }
+}
diff --git a/internal/scrape/render.go b/internal/scrape/render.go
index 3f24ba59..eb5171f6 100644
--- a/internal/scrape/render.go
+++ b/internal/scrape/render.go
@@ -5,6 +5,16 @@ import (
"strings"
)
+// SampledPagesSummary describes how many pages a run sampled, naming the
+// sitemap total only when a sitemap supplied one. Shared with the CLI preview
+// so both surfaces use the same nouns for the same two numbers.
+func SampledPagesSummary(sampled, totalURLsInSitemap int) string {
+ if totalURLsInSitemap > 0 {
+ return fmt.Sprintf("%d page(s) sampled of %d in sitemap", sampled, totalURLsInSitemap)
+ }
+ return fmt.Sprintf("%d page(s) sampled", sampled)
+}
+
// RenderMarkdown renders the report as a single Markdown digest: site
// overview, the page tree by URL pattern, then each page's content.
func RenderMarkdown(r Report) []byte {
@@ -15,7 +25,7 @@ func RenderMarkdown(r Report) []byte {
}
fmt.Fprintf(&b, "# Scrape Report — %s\n\n", title)
fmt.Fprintf(&b, "- Base URL: %s\n", r.Site.BaseURL)
- fmt.Fprintf(&b, "- Pages: %d sampled of %d discovered\n", r.Site.SampledPages, r.Site.TotalDiscovered)
+ fmt.Fprintf(&b, "- Pages: %s\n", SampledPagesSummary(r.Site.SampledPages, r.Site.TotalURLsInSitemap))
fmt.Fprintf(&b, "- Sources: %d http · %d browser-rendered · %d failed\n",
r.Summary.HTTPPages, r.Summary.BrowserPages, r.Summary.FailedPages)
fmt.Fprintf(&b, "- Generated: %s\n\n", r.GeneratedAt.Format("2006-01-02 15:04:05 UTC"))
diff --git a/internal/scrape/run.go b/internal/scrape/run.go
index 9f3ef9ac..92c2d19d 100644
--- a/internal/scrape/run.go
+++ b/internal/scrape/run.go
@@ -130,7 +130,7 @@ func URLListCrawler(urls []string, timeout time.Duration, guard CrawlGuard) Craw
pages = append(pages, fetchOne(ctx, u, timeout, policy))
}
return &seaportal.ScrapeResult{
- Site: seaportal.SiteInfo{BaseURL: originOf(clean[0]), TotalURLsInSitemap: len(clean)},
+ Site: seaportal.SiteInfo{BaseURL: originOf(clean[0])},
Pages: pages,
}, nil
}
@@ -240,11 +240,11 @@ func Run(ctx context.Context, input Input, opts RunOptions, crawl Crawler, rende
GeneratedAt: time.Now().UTC(),
Input: input,
Site: SiteInfo{
- BaseURL: res.Site.BaseURL,
- Title: res.Site.Title,
- SitemapFound: res.Site.SitemapFound,
- TotalDiscovered: res.Site.TotalURLsInSitemap,
- SampledPages: len(pages),
+ BaseURL: res.Site.BaseURL,
+ Title: res.Site.Title,
+ SitemapFound: res.Site.SitemapFound,
+ TotalURLsInSitemap: res.Site.TotalURLsInSitemap,
+ SampledPages: len(pages),
},
PageGroups: fromSeaportalGroups(res.PageGroups),
Pages: pages,
diff --git a/internal/scrape/run_test.go b/internal/scrape/run_test.go
index 64071123..1cf0fd30 100644
--- a/internal/scrape/run_test.go
+++ b/internal/scrape/run_test.go
@@ -362,3 +362,95 @@ func TestRunConcurrencyIsBounded(t *testing.T) {
t.Fatalf("Run: %v", err)
}
}
+
+// crawlWithoutSitemap is the shape every site without a usable sitemap
+// produces: pages found by crawling, no sitemap total to report.
+func crawlWithoutSitemap(pages ...seaportal.PageObject) Crawler {
+ return func(context.Context) (*seaportal.ScrapeResult, error) {
+ return &seaportal.ScrapeResult{
+ Site: seaportal.SiteInfo{BaseURL: "https://example.com", SitemapFound: false},
+ Pages: pages,
+ }, nil
+ }
+}
+
+func TestRunWithoutSitemapNeverReportsATotalBelowTheSampledPages(t *testing.T) {
+ crawl := crawlWithoutSitemap(
+ seaportal.PageObject{URL: "https://example.com/", Status: 200, Markdown: longMarkdown},
+ seaportal.PageObject{URL: "https://example.com/docs", Status: 200, Markdown: longMarkdown},
+ )
+
+ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{Preview: true}, crawl, nil)
+ if err != nil {
+ t.Fatalf("Run: %v", err)
+ }
+
+ if report.Site.TotalURLsInSitemap != 0 {
+ t.Fatalf("TotalURLsInSitemap = %d, want 0 without a sitemap", report.Site.TotalURLsInSitemap)
+ }
+ summary := SampledPagesSummary(len(report.Pages), report.Site.TotalURLsInSitemap)
+ if strings.Contains(summary, " of ") {
+ t.Fatalf("summary %q claims a total the crawl never had", summary)
+ }
+ if !strings.Contains(summary, fmt.Sprintf("%d page(s) sampled", len(report.Pages))) {
+ t.Fatalf("summary %q does not report the %d pages it lists", summary, len(report.Pages))
+ }
+ if md := string(RenderMarkdown(report)); !strings.Contains(md, summary) || strings.Contains(md, "of 0") {
+ t.Fatalf("markdown report disagrees with the preview line: %q", firstLines(md, 4))
+ }
+}
+
+func TestRunWithSitemapStillReportsTheSitemapTotal(t *testing.T) {
+ pages := []seaportal.PageObject{
+ {URL: "https://example.com/", Status: 200, Markdown: longMarkdown},
+ {URL: "https://example.com/docs", Status: 200, Markdown: longMarkdown},
+ }
+ crawl := func(context.Context) (*seaportal.ScrapeResult, error) {
+ return &seaportal.ScrapeResult{
+ Site: seaportal.SiteInfo{BaseURL: "https://example.com", SitemapFound: true, TotalURLsInSitemap: 42},
+ Pages: pages,
+ }, nil
+ }
+
+ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{Preview: true}, crawl, nil)
+ if err != nil {
+ t.Fatalf("Run: %v", err)
+ }
+
+ if !report.Site.SitemapFound || report.Site.TotalURLsInSitemap != 42 {
+ t.Fatalf("sitemap info lost: found=%v total=%d", report.Site.SitemapFound, report.Site.TotalURLsInSitemap)
+ }
+ summary := SampledPagesSummary(len(report.Pages), report.Site.TotalURLsInSitemap)
+ if !strings.Contains(summary, "42 in sitemap") {
+ t.Fatalf("summary %q drops the sitemap total", summary)
+ }
+ if md := string(RenderMarkdown(report)); !strings.Contains(md, summary) {
+ t.Fatalf("markdown report disagrees with the preview line: %q", firstLines(md, 4))
+ }
+}
+
+func TestURLListCrawlerLeavesTheSitemapTotalUnset(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ _, _ = fmt.Fprint(w, renderedHTML)
+ }))
+ defer srv.Close()
+ guard := CrawlGuard{ValidateURL: func(string) error { return nil }}
+ crawl := URLListCrawler([]string{srv.URL + "/a", srv.URL + "/b"}, 5*time.Second, guard)
+
+ res, err := crawl(context.Background())
+ if err != nil {
+ t.Fatalf("crawl: %v", err)
+ }
+ if res.Site.TotalURLsInSitemap != 0 {
+ t.Fatalf("TotalURLsInSitemap = %d, want 0: caller-supplied URLs never came from a sitemap", res.Site.TotalURLsInSitemap)
+ }
+}
+
+func firstLines(s string, n int) string {
+ lines := strings.Split(s, "\n")
+ if len(lines) > n {
+ lines = lines[:n]
+ }
+ return strings.Join(lines, "\n")
+}
diff --git a/internal/scrape/types.go b/internal/scrape/types.go
index 03055c71..b11fb442 100644
--- a/internal/scrape/types.go
+++ b/internal/scrape/types.go
@@ -9,7 +9,7 @@ import "time"
// SchemaVersion is the current scrape report schema version, embedded in
// every report so consumers can detect format changes.
-const SchemaVersion = "1.0"
+const SchemaVersion = "2.0"
// Page sources: how a page's content was obtained.
const (
@@ -73,11 +73,11 @@ type PageGroup struct {
// SiteInfo describes the crawled site.
type SiteInfo struct {
- BaseURL string `json:"baseUrl"`
- Title string `json:"title,omitempty"`
- SitemapFound bool `json:"sitemapFound"`
- TotalDiscovered int `json:"totalDiscovered"`
- SampledPages int `json:"sampledPages"`
+ BaseURL string `json:"baseUrl"`
+ Title string `json:"title,omitempty"`
+ SitemapFound bool `json:"sitemapFound"`
+ TotalURLsInSitemap int `json:"totalURLsInSitemap,omitempty"`
+ SampledPages int `json:"sampledPages"`
}
// Summary is the run roll-up.