Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick:

Nicolas22848af5ae79
diff --git a/apps/api/src/__tests__/e2e_extract/index.test.ts b/apps/api/src/__tests__/e2e_extract/index.test.ts
index ec68ca125..f97b111e3 100644
--- a/apps/api/src/__tests__/e2e_extract/index.test.ts
+++ b/apps/api/src/__tests__/e2e_extract/index.test.ts
@@ -35,6 +35,9 @@ describe("E2E Tests for Extract API Routes", () => {
if (author.includes("Gergő Móricz")) gotItRight++;
if (author.includes("Eric Ciarla")) gotItRight++;
if (author.includes("Nicolas Camara")) gotItRight++;
+ if (author.includes("Jon")) gotItRight++;
+ if (author.includes("Wendong")) gotItRight++;
+
}
expect(gotItRight).toBeGreaterThan(1);
@@ -46,7 +49,7 @@ describe("E2E Tests for Extract API Routes", () => {
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
- urls: ["mendable.ai/*"],
+ urls: ["firecrawl.dev/*"],
prompt: "Who are the founders of the company?",
allowExternalLinks: true,
schema: {
@@ -58,14 +61,16 @@ describe("E2E Tests for Extract API Routes", () => {
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("founders");
+ console.log(response.body.data?.founders);
let gotItRight = 0;
for (const founder of response.body.data?.founders) {
if (founder.includes("Caleb")) gotItRight++;
if (founder.includes("Eric")) gotItRight++;
if (founder.includes("Nicolas")) gotItRight++;
+
}
- expect(gotItRight).toBe(3);
+ expect(gotItRight).toBeGreaterThanOrEqual(2);
}, 60000);
it.concurrent("should return hiring opportunities on firecrawl.dev (allowExternalLinks = true)", async () => {
@@ -90,15 +95,12 @@ describe("E2E Tests for Extract API Routes", () => {
let gotItRight = 0;
for (const hiring of response.body.data?.items) {
- if (hiring.includes("Developer Relations Specialist")) gotItRight++;
- if (hiring.includes("Web Automation Engineer")) gotItRight++;
- if (hiring.includes("Developer Experience Engineer")) gotItRight++;
if (hiring.includes("Developer Support Engineer")) gotItRight++;
if (hiring.includes("Dev Ops Engineer")) gotItRight++;
if (hiring.includes("Founding Web Automation Engineer")) gotItRight++;
}
- expect(gotItRight).toBeGreaterThan(5);
+ expect(gotItRight).toBeGreaterThan(2);
}, 60000);
it.concurrent("should return PCI DSS compliance for Fivetran", async () => {
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 0a94289ce..1513365de 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -29,7 +29,7 @@ const redis = new Redis(process.env.REDIS_URL!);
const MAX_EXTRACT_LIMIT = 100;
const MAX_RANKING_LIMIT = 10;
-const SCORE_THRESHOLD = 0.70;
+const SCORE_THRESHOLD = 0.75;
export async function extractController(
req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
@@ -107,7 +107,6 @@ export async function extractController(
links.push(...processedUrls.flat());
console.log("links", links.length);
- console
// Scrape all links in parallel
const scrapePromises = links.map(async (url) => {
const origin = req.body.origin || "api";