Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(map): do not remove query parameters from results (FIR-1015) (#1191)
Gergő Móricz7ecbff3b20fb
diff --git a/apps/api/src/__tests__/snips/map.test.ts b/apps/api/src/__tests__/snips/map.test.tsindex be1469ffc..e156a3f3e 100644--- a/apps/api/src/__tests__/snips/map.test.ts+++ b/apps/api/src/__tests__/snips/map.test.ts@@ -39,4 +39,24 @@ describe("Map tests", () => {expect(response.body.success).toBe(false);expect(response.body.error).toBe("Request timed out");}, 10000);++ it("handles query parameters correctly", async () => {+ let response = await map({+ url: "https://www.hfea.gov.uk",+ sitemapOnly: true,+ });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(response.body.links.some(x => x.match(/^https:\/\/www\.hfea\.gov\.uk\/choose-a-clinic\/clinic-search\/results\/?\?options=\d+$/))).toBe(true);++ response = await map({+ url: "https://www.hfea.gov.uk",+ ignoreSitemap: false,+ });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(response.body.links.some(x => x.match(/^https:\/\/www\.hfea\.gov\.uk\/choose-a-clinic\/clinic-search\/results\/?\?options=\d+$/))).toBe(true);+ }, 300000); // TODO: mocks});diff --git a/apps/api/src/lib/validateUrl.ts b/apps/api/src/lib/validateUrl.tsindex dc27c1363..d7672bb95 100644--- a/apps/api/src/lib/validateUrl.ts+++ b/apps/api/src/lib/validateUrl.ts@@ -147,7 +147,7 @@ export const checkAndUpdateURLForMap = (url: string) => {}// remove any query params- url = url.split("?")[0].trim();+ // url = url.split("?")[0].trim();return { urlObj: typedUrlObj, url: url };};