Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: no sections edge case (#1814)
Nicolas51a7ad4a9072
diff --git a/apps/api/src/scraper/WebScraper/__tests__/utils.test.ts b/apps/api/src/scraper/WebScraper/__tests__/utils.test.tsnew file mode 100644index 000000000..b63401053--- /dev/null+++ b/apps/api/src/scraper/WebScraper/__tests__/utils.test.ts@@ -0,0 +1,45 @@+import { WebCrawler } from '../crawler';++describe('WebCrawler - noSections', () => {+ let crawler: WebCrawler;++ beforeEach(() => {+ crawler = new WebCrawler({+ jobId: 'test-job',+ initialUrl: 'https://example.com',+ baseUrl: 'https://example.com',+ includes: [],+ excludes: [],+ });+ });++ describe('noSections method', () => {+ it('should return true for URLs without hash fragments', () => {+ expect(crawler['noSections']('https://example.com/page')).toBe(true);+ expect(crawler['noSections']('https://example.com/blog/post')).toBe(true);+ expect(crawler['noSections']('https://example.com')).toBe(true);+ });++ it('should return false for simple anchor links', () => {+ expect(crawler['noSections']('https://example.com/page#section')).toBe(false);+ expect(crawler['noSections']('https://example.com/page#top')).toBe(false);+ expect(crawler['noSections']('https://example.com/page#')).toBe(false);+ expect(crawler['noSections']('https://example.com/page#a')).toBe(false);+ });++ it('should return true for hash fragments that look like routes', () => {+ expect(crawler['noSections']('https://example.com/app#/dashboard')).toBe(true);+ expect(crawler['noSections']('https://example.com/spa#/user/profile')).toBe(true);+ expect(crawler['noSections']('https://example.com/page#/settings/account')).toBe(true);+ });++ it('should return false for short hash fragments even with slashes', () => {+ expect(crawler['noSections']('https://example.com/page#/')).toBe(false);+ });++ it('should handle edge cases', () => {+ expect(crawler['noSections']('https://example.com/page#ab')).toBe(false);+ expect(crawler['noSections']('https://example.com/page#abc/def')).toBe(true);+ });+ });+});diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 390bf81e4..4b5942e77 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -628,7 +628,19 @@ export class WebCrawler {}private noSections(link: string): boolean {- return !link.includes("#");+ // Allow URLs with hash fragments that represent actual routes/pages (like SPAs)+ // but block simple anchor links within the same page+ if (!link.includes("#")) {+ return true;+ }++ // Check if the hash fragment looks like a route (contains forward slashes and has substantial content)+ const hashPart = link.split("#")[1];+ if (hashPart && hashPart.length > 1 && hashPart.includes("/")) {+ return true;+ }++ return false;}private isInternalLink(link: string): boolean {