Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas22848af5ae79
diff --git a/apps/api/src/__tests__/e2e_extract/index.test.ts b/apps/api/src/__tests__/e2e_extract/index.test.tsindex ec68ca125..f97b111e3 100644--- a/apps/api/src/__tests__/e2e_extract/index.test.ts+++ b/apps/api/src/__tests__/e2e_extract/index.test.ts@@ -35,6 +35,9 @@ describe("E2E Tests for Extract API Routes", () => {if (author.includes("Gergő Móricz")) gotItRight++;if (author.includes("Eric Ciarla")) gotItRight++;if (author.includes("Nicolas Camara")) gotItRight++;+ if (author.includes("Jon")) gotItRight++;+ if (author.includes("Wendong")) gotItRight++;+}expect(gotItRight).toBeGreaterThan(1);@@ -46,7 +49,7 @@ describe("E2E Tests for Extract API Routes", () => {.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`).set("Content-Type", "application/json").send({- urls: ["mendable.ai/*"],+ urls: ["firecrawl.dev/*"],prompt: "Who are the founders of the company?",allowExternalLinks: true,schema: {@@ -58,14 +61,16 @@ describe("E2E Tests for Extract API Routes", () => {expect(response.body).toHaveProperty("data");expect(response.body.data).toHaveProperty("founders");+ console.log(response.body.data?.founders);let gotItRight = 0;for (const founder of response.body.data?.founders) {if (founder.includes("Caleb")) gotItRight++;if (founder.includes("Eric")) gotItRight++;if (founder.includes("Nicolas")) gotItRight++;+}- expect(gotItRight).toBe(3);+ expect(gotItRight).toBeGreaterThanOrEqual(2);}, 60000);it.concurrent("should return hiring opportunities on firecrawl.dev (allowExternalLinks = true)", async () => {@@ -90,15 +95,12 @@ describe("E2E Tests for Extract API Routes", () => {let gotItRight = 0;for (const hiring of response.body.data?.items) {- if (hiring.includes("Developer Relations Specialist")) gotItRight++;- if (hiring.includes("Web Automation Engineer")) gotItRight++;- if (hiring.includes("Developer Experience Engineer")) gotItRight++;if (hiring.includes("Developer Support Engineer")) gotItRight++;if (hiring.includes("Dev Ops Engineer")) gotItRight++;if (hiring.includes("Founding Web Automation Engineer")) gotItRight++;}- expect(gotItRight).toBeGreaterThan(5);+ expect(gotItRight).toBeGreaterThan(2);}, 60000);it.concurrent("should return PCI DSS compliance for Fivetran", async () => {diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 0a94289ce..1513365de 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -29,7 +29,7 @@ const redis = new Redis(process.env.REDIS_URL!);const MAX_EXTRACT_LIMIT = 100;const MAX_RANKING_LIMIT = 10;-const SCORE_THRESHOLD = 0.70;+const SCORE_THRESHOLD = 0.75;export async function extractController(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,@@ -107,7 +107,6 @@ export async function extractController(links.push(...processedUrls.flat());console.log("links", links.length);- console// Scrape all links in parallelconst scrapePromises = links.map(async (url) => {const origin = req.body.origin || "api";