High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: scrape forwards the HTTP crawler's pre-render recommendations, so the report calls an enriched page thin and advises enrichment it already did (PIN-205)
Luigi Agosti3793c5152160
diff --git a/internal/scrape/run.go b/internal/scrape/run.goindex 92c2d19d..4764dbfb 100644--- a/internal/scrape/run.go+++ b/internal/scrape/run.go@@ -318,8 +318,37 @@ func fromSeaportalGroups(groups []seaportal.PageGroup) []PageGroup {return out}-func summarize(pages []Page, recommendations []string) Summary {- s := Summary{ContentTypes: map[string]int{}, Recommendations: recommendations}+// ThinContentChars is the post-enrichment thin-content threshold: a page whose final+// extraction is shorter than this has little usable text.+//+// PinchTab OWNS this number. seaportal applies the same 160 at the pinned version, but+// its constant is unexported AND inside another module's internal/ tree, so no import,+// linkname or reflection can reach it — claiming to mirror it would be a promise this+// code cannot keep. Owning it means a future upstream change becomes a visible decision+// here rather than a silent divergence with nothing failing.+const ThinContentChars = 160++// forwardableRecommendations are the upstream recommendations that survive the browser+// phase, matched on the stable noun in each.+//+// THE RULE, recorded where the choice is made, so the next upstream recommendation is+// classified rather than guessed at: a recommendation derived from PAGE CONTENT must be+// regenerated after enrichment, because the browser phase changes exactly that; one+// derived from CRAWL SCOPE is forwarded, because the browser phase changes nothing about+// which URLs were sampled. Both sitemap lines are scope; the errors and thin-content+// lines are content, and both are regenerated below from the final pages.+//+// An unrecognised recommendation is FORWARDED, since dropping advice that is still true+// is the worse failure. Classify a new one here when it appears.+var forwardableRecommendations = []string{"sitemap"}++// summarize rolls the FINAL pages up. Every field including the prose comes from this one+// slice: the recommendations used to be carried in verbatim from the HTTP crawl, so the+// numbers were post-render and the sentences pre-render inside one object — a run that+// enriched every page still advised the reader to consider enrichment, and counted a page+// as having little text while reporting its 2012 characters two lines above.+func summarize(pages []Page, inherited []string) Summary {+ s := Summary{ContentTypes: map[string]int{}}for _, p := range pages {if p.ContentType != "" {s.ContentTypes[p.ContentType]++@@ -336,9 +365,73 @@ func summarize(pages []Page, recommendations []string) Summary {if len(s.ContentTypes) == 0 {s.ContentTypes = nil}+ s.Recommendations = recommend(pages, s, inherited)return s}+// recommend keeps the crawl-scope advice the HTTP phase produced and regenerates the+// page-content advice from the pages as they finally stand.+func recommend(pages []Page, s Summary, inherited []string) []string {+ var recs []string+ if s.FailedPages > 0 {+ recs = append(recs, fmt.Sprintf("%d of %d pages returned errors or 4xx/5xx responses", s.FailedPages, len(pages)))+ }+ if thin, unenriched := thinPages(pages); thin > 0 {+ // The advice to enrich only survives while it is still actionable. A page the+ // browser already rendered — or tried to and failed — has had that remedy+ // applied, so repeating it sends the reader on a re-run that changes nothing.+ line := fmt.Sprintf("%d pages have little extractable text (possible SPA/JS-only)", thin)+ if unenriched > 0 {+ line += "; consider PinchTab enrichment"+ }+ recs = append(recs, line)+ }+ for _, rec := range inherited {+ if !regeneratedLocally(rec) {+ recs = append(recs, rec)+ }+ }+ return recs+}++// thinPages counts the pages whose FINAL content is thin, and how many of those never+// had a browser attempt — the second number is what decides whether advising enrichment+// is still useful. A failed page is not also thin: it has no content for a reason that+// the errors line already reports.+func thinPages(pages []Page) (thin, unenriched int) {+ for _, p := range pages {+ if p.Error != "" || p.StatusCode >= 400 {+ continue+ }+ if contentChars(p) >= ThinContentChars {+ continue+ }+ thin+++ if p.Source != SourceBrowser && p.BrowserError == "" {+ unenriched+++ }+ }+ return thin, unenriched+}++// contentChars is a page's extracted length, read from CharCount in preview mode where+// the body is deliberately withheld and Markdown is empty for every page.+func contentChars(p Page) int {+ if p.Markdown != "" {+ return utf8.RuneCountInString(strings.TrimSpace(p.Markdown))+ }+ return p.CharCount+}++func regeneratedLocally(rec string) bool {+ for _, forwardable := range forwardableRecommendations {+ if strings.Contains(rec, forwardable) {+ return false+ }+ }+ return true+}+// clampConcurrency normalizes a requested concurrency into [1, MaxConcurrency].func clampConcurrency(n int) int {switch {diff --git a/internal/scrape/run_test.go b/internal/scrape/run_test.goindex 1cf0fd30..49592a78 100644--- a/internal/scrape/run_test.go+++ b/internal/scrape/run_test.go@@ -2,6 +2,7 @@ package scrapeimport ("context"+ "encoding/json""errors""fmt""net/http"@@ -454,3 +455,261 @@ func firstLines(s string, n int) string {}return strings.Join(lines, "\n")}++// The upstream crawl's own recommendations, which PinchTab used to forward verbatim.+// They are written for a caller who has NOT yet run a browser, and PinchTab is a caller+// who just did.+const (+ inheritedThinRec = "2 pages have little extractable text (possible SPA/JS-only); consider PinchTab enrichment"+ inheritedErrorsRec = "1 of 4 pages returned errors or 4xx/5xx responses"+ inheritedSitemapRec = "sitemap lists 400 URLs across 12 patterns but only 4 were sampled; sampling recommended"+ inheritedCoverRec = "only 4 of 400 sitemap URLs were sampled (1.0% coverage); raise --max-pages for broader coverage"+)++func crawlAdvising(recs []string, pages ...seaportal.PageObject) Crawler {+ return func(context.Context) (*seaportal.ScrapeResult, error) {+ return &seaportal.ScrapeResult{+ Site: seaportal.SiteInfo{BaseURL: "https://example.com", TotalURLsInSitemap: len(pages)},+ Pages: pages,+ Summary: seaportal.ScrapeSummary{Recommendations: recs},+ PageGroups: []seaportal.PageGroup{+ {Pattern: "/*", TotalInSitemap: len(pages), Sampled: len(pages), Pages: pages},+ },+ }, nil+ }+}++func recommendationNaming(t *testing.T, report Report, substr string) string {+ t.Helper()+ for _, rec := range report.Summary.Recommendations {+ if strings.Contains(rec, substr) {+ return rec+ }+ }+ return ""+}++// The discriminating fixture: a JS-only page that extracts to nothing over HTTP and to+// plenty after rendering, beside a page that is thin either way. The pre-render count is+// 2 and the post-render count is 1, so a recommendation carried over from the HTTP phase+// says 2 while the report's own per-page records say otherwise.+func TestThinRecommendationCountsPagesAsTheyFinallyStand(t *testing.T) {+ crawl := crawlAdvising([]string{inheritedThinRec},+ seaportal.PageObject{URL: "https://example.com/", Status: 200, Markdown: longMarkdown, ContentType: "page"},+ seaportal.PageObject{URL: "https://example.com/about", Status: 200, Markdown: longMarkdown, ContentType: "page"},+ seaportal.PageObject{URL: "https://example.com/js", Status: 200, Markdown: "", ContentType: "page"},+ seaportal.PageObject{URL: "https://example.com/thin", Status: 200, Markdown: "hi", ContentType: "page"},+ )+ render := func(url string) (string, error) {+ if strings.HasSuffix(url, "/thin") {+ return `<html><body><article><p>hi</p></article></body></html>`, nil+ }+ return renderedHTML, nil+ }++ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{}, crawl, render)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ js := report.Pages[2]+ if js.Source != SourceBrowser || utf8.RuneCountInString(js.Markdown) < ThinContentChars {+ t.Fatalf("the JS page was not filled by the browser, so this fixture cannot tell the two counts apart: source=%q chars=%d", js.Source, utf8.RuneCountInString(js.Markdown))+ }++ thinRec := recommendationNaming(t, report, "little extractable text")+ if thinRec == "" {+ t.Fatalf("the still-thin page produced no thin-content recommendation: %v", report.Summary.Recommendations)+ }+ if !strings.HasPrefix(thinRec, "1 pages") {+ t.Errorf("thin count is pre-render: %q — the browser filled one of the two pages the HTTP phase called thin", thinRec)+ }+ if strings.Contains(thinRec, "consider PinchTab enrichment") {+ t.Errorf("the report advises the enrichment it already performed: %q", thinRec)+ }+}++// --enrich-all renders every page by definition, so advising enrichment can never be+// actionable — a reader who follows it re-runs and gets the same sentence back.+func TestEnrichAllNeverAdvisesEnrichment(t *testing.T) {+ crawl := crawlAdvising([]string{inheritedThinRec},+ seaportal.PageObject{URL: "https://example.com/", Status: 200, Markdown: longMarkdown},+ seaportal.PageObject{URL: "https://example.com/thin", Status: 200, Markdown: "hi"},+ )+ render := func(url string) (string, error) {+ if strings.HasSuffix(url, "/thin") {+ return `<html><body><article><p>hi</p></article></body></html>`, nil+ }+ return renderedHTML, nil+ }++ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{EnrichAll: true}, crawl, render)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ if report.Summary.BrowserPages == 0 {+ t.Fatal("no page was browser-rendered, so --enrich-all is not being exercised")+ }+ for _, rec := range report.Summary.Recommendations {+ if strings.Contains(rec, "consider PinchTab enrichment") {+ t.Errorf("--enrich-all still advises enrichment: %q", rec)+ }+ }+ if recommendationNaming(t, report, "little extractable text") == "" {+ t.Error("the page that is thin even after rendering stopped being reported; that half is still worth saying")+ }+}++// A page that is thin and was never offered to the browser keeps the advice, because+// there the remedy has not been applied yet.+func TestThinPagesTheBrowserNeverSawKeepTheEnrichmentAdvice(t *testing.T) {+ crawl := crawlAdvising([]string{inheritedThinRec},+ seaportal.PageObject{URL: "https://example.com/thin", Status: 200, Markdown: "hi"},+ )++ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{NoBrowser: true}, crawl, nil)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ thinRec := recommendationNaming(t, report, "little extractable text")+ if !strings.Contains(thinRec, "consider PinchTab enrichment") {+ t.Errorf("with no browser phase the advice is still actionable and must survive: %q", thinRec)+ }+}++// The second live instance of the same defect: the browser can recover a page that failed+// over HTTP, and summarize already counts FailedPages from the final pages — so the+// inherited errors line contradicts the report's own number.+func TestErrorsRecommendationCountsPagesAsTheyFinallyStand(t *testing.T) {+ crawl := crawlAdvising([]string{inheritedErrorsRec},+ seaportal.PageObject{URL: "https://example.com/", Status: 200, Markdown: longMarkdown},+ seaportal.PageObject{URL: "https://example.com/flaky", Status: 200, Error: "tls handshake failure"},+ )+ render := func(string) (string, error) { return renderedHTML, nil }++ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{}, crawl, render)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ recovered := report.Pages[1]+ if recovered.Error != "" || recovered.Source != SourceBrowser {+ t.Fatalf("the browser did not recover the failed page, so this fixture proves nothing: error=%q source=%q", recovered.Error, recovered.Source)+ }+ if report.Summary.FailedPages != 0 {+ t.Fatalf("FailedPages = %d, want 0 after recovery", report.Summary.FailedPages)+ }+ if rec := recommendationNaming(t, report, "returned errors"); rec != "" {+ t.Errorf("the report says %q while its own failedPages is 0", rec)+ }+}++func TestAGenuinelyFailedPageIsStillReported(t *testing.T) {+ crawl := crawlAdvising(nil,+ seaportal.PageObject{URL: "https://example.com/", Status: 200, Markdown: longMarkdown},+ seaportal.PageObject{URL: "https://example.com/dead", Status: 200, Error: "tls handshake failure"},+ )+ render := func(string) (string, error) { return "", errors.New("tab crashed") }++ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{}, crawl, render)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ rec := recommendationNaming(t, report, "returned errors")+ if rec == "" {+ t.Fatalf("a page that failed and stayed failed produced no recommendation: %v", report.Summary.Recommendations)+ }+ if !strings.HasPrefix(rec, "1 of 2 pages") {+ t.Errorf("the errors line does not match the report's own failedPages=%d: %q", report.Summary.FailedPages, rec)+ }+}++// Crawl-scope advice describes which URLs were SAMPLED, which no amount of browser+// rendering changes, so it must survive verbatim. Dropping advice that is still true is+// the failure mode on the other side of this fix.+func TestSitemapCoverageAdviceIsForwardedUnchanged(t *testing.T) {+ crawl := crawlAdvising([]string{inheritedSitemapRec, inheritedCoverRec},+ seaportal.PageObject{URL: "https://example.com/", Status: 200, Markdown: longMarkdown},+ )++ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{NoBrowser: true}, crawl, nil)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ for _, want := range []string{inheritedSitemapRec, inheritedCoverRec} {+ found := false+ for _, rec := range report.Summary.Recommendations {+ if rec == want {+ found = true+ }+ }+ if !found {+ t.Errorf("crawl-scope advice was dropped: %q\n got %v", want, report.Summary.Recommendations)+ }+ }+}++// A clean run says nothing rather than saying nothing under a heading: an empty+// "## Recommendations" section reads as a report with no conclusions.+func TestACleanRunOmitsTheRecommendationsBlockEntirely(t *testing.T) {+ crawl := crawlAdvising([]string{inheritedThinRec, inheritedErrorsRec},+ seaportal.PageObject{URL: "https://example.com/", Status: 200, Markdown: longMarkdown},+ seaportal.PageObject{URL: "https://example.com/js", Status: 200, Markdown: ""},+ )+ render := func(string) (string, error) { return renderedHTML, nil }++ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{EnrichAll: true}, crawl, render)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ if len(report.Summary.Recommendations) != 0 {+ t.Fatalf("a run with nothing left to advise still carries %v", report.Summary.Recommendations)+ }+ encoded, err := json.Marshal(report)+ if err != nil {+ t.Fatal(err)+ }+ if strings.Contains(string(encoded), "recommendations") {+ t.Error("the JSON report carries an empty recommendations key; omitempty needs a nil slice, not an allocated one")+ }+ if strings.Contains(string(RenderMarkdown(report)), "## Recommendations") {+ t.Error("the markdown report renders a Recommendations heading with nothing under it")+ }+}++// The threshold is PinchTab's own. seaportal applies the same number at the pinned+// version, but its constant is unexported and inside another module's internal/ tree, so+// nothing here can read it — this test is the pin that makes a future divergence a+// decision someone takes rather than one that happens.+func TestThinContentThresholdIsPinned(t *testing.T) {+ if ThinContentChars != 160 {+ t.Errorf("ThinContentChars = %d, want 160 — the value matches seaportal's own thin-content threshold at the pinned version. Changing it is allowed, but say why in the constant's comment, because the two notions of thin then differ", ThinContentChars)+ }+}++// Preview withholds every page body, so a thin count read from Markdown would call the+// whole site thin.+func TestPreviewCountsWithheldBodiesByTheirRecordedLength(t *testing.T) {+ crawl := crawlAdvising(nil,+ seaportal.PageObject{URL: "https://example.com/", Status: 200, Markdown: longMarkdown},+ seaportal.PageObject{URL: "https://example.com/thin", Status: 200, Markdown: "hi"},+ )++ report, err := Run(context.Background(), Input{URL: "https://example.com"}, RunOptions{Preview: true}, crawl, nil)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ if report.Pages[0].Markdown != "" || report.Pages[0].CharCount == 0 {+ t.Fatalf("preview did not withhold the body, so this fixture proves nothing: %+v", report.Pages[0])+ }+ thinRec := recommendationNaming(t, report, "little extractable text")+ if !strings.HasPrefix(thinRec, "1 pages") {+ t.Errorf("preview counted withheld bodies as thin: %q", thinRec)+ }+}