Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: no sections edge case (#1814)

Nicolas51a7ad4a9072
diff --git a/apps/api/src/scraper/WebScraper/__tests__/utils.test.ts b/apps/api/src/scraper/WebScraper/__tests__/utils.test.ts
new file mode 100644
index 000000000..b63401053
--- /dev/null
+++ b/apps/api/src/scraper/WebScraper/__tests__/utils.test.ts
@@ -0,0 +1,45 @@
+import { WebCrawler } from '../crawler';
+
+describe('WebCrawler - noSections', () => {
+ let crawler: WebCrawler;
+
+ beforeEach(() => {
+ crawler = new WebCrawler({
+ jobId: 'test-job',
+ initialUrl: 'https://example.com',
+ baseUrl: 'https://example.com',
+ includes: [],
+ excludes: [],
+ });
+ });
+
+ describe('noSections method', () => {
+ it('should return true for URLs without hash fragments', () => {
+ expect(crawler['noSections']('https://example.com/page')).toBe(true);
+ expect(crawler['noSections']('https://example.com/blog/post')).toBe(true);
+ expect(crawler['noSections']('https://example.com')).toBe(true);
+ });
+
+ it('should return false for simple anchor links', () => {
+ expect(crawler['noSections']('https://example.com/page#section')).toBe(false);
+ expect(crawler['noSections']('https://example.com/page#top')).toBe(false);
+ expect(crawler['noSections']('https://example.com/page#')).toBe(false);
+ expect(crawler['noSections']('https://example.com/page#a')).toBe(false);
+ });
+
+ it('should return true for hash fragments that look like routes', () => {
+ expect(crawler['noSections']('https://example.com/app#/dashboard')).toBe(true);
+ expect(crawler['noSections']('https://example.com/spa#/user/profile')).toBe(true);
+ expect(crawler['noSections']('https://example.com/page#/settings/account')).toBe(true);
+ });
+
+ it('should return false for short hash fragments even with slashes', () => {
+ expect(crawler['noSections']('https://example.com/page#/')).toBe(false);
+ });
+
+ it('should handle edge cases', () => {
+ expect(crawler['noSections']('https://example.com/page#ab')).toBe(false);
+ expect(crawler['noSections']('https://example.com/page#abc/def')).toBe(true);
+ });
+ });
+});
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 390bf81e4..4b5942e77 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -628,7 +628,19 @@ export class WebCrawler {
}
private noSections(link: string): boolean {
- return !link.includes("#");
+ // Allow URLs with hash fragments that represent actual routes/pages (like SPAs)
+ // but block simple anchor links within the same page
+ if (!link.includes("#")) {
+ return true;
+ }
+
+ // Check if the hash fragment looks like a route (contains forward slashes and has substantial content)
+ const hashPart = link.split("#")[1];
+ if (hashPart && hashPart.length > 1 && hashPart.includes("/")) {
+ return true;
+ }
+
+ return false;
}
private isInternalLink(link: string): boolean {