Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

mvp done?

rafaelmmiller25f32000dbff
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 6f7aced13..d3c06c34a 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -89,7 +89,7 @@ export async function extractController(
await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours
}
- console.log("allResults", allResults);
+ // console.log("allResults", allResults);
// Parallelize sitemap fetch with serper search
const [sitemap, ...searchResults] = await Promise.all([
req.body.ignoreSitemap ? null : crawler.tryGetSitemap(),
@@ -138,7 +138,7 @@ export async function extractController(
linksAndScores = await performRanking(links, searchQuery);
}
- console.log("linksAndScores", linksAndScores);
+ // console.log("linksAndScores", linksAndScores);
links = links
.map((x) => {
@@ -218,8 +218,8 @@ export async function extractController(
await getScrapeQueue().remove(jobId);
- const endTime = new Date().getTime();
- const timeTakenInSeconds = (endTime - startTime) / 1000;
+ // const endTime = new Date().getTime();
+ // const timeTakenInSeconds = (endTime - startTime) / 1000;
// const numTokens =
// doc && doc.extract
// // ? numTokensFromString(doc.markdown, "gpt-3.5-turbo")
@@ -235,7 +235,7 @@ export async function extractController(
}
- console.log("docs", docs);
+ // console.log("docs", docs);
// {"message":"Missing required parameter: 'response_format.json_schema.schema'.","type":"invalid_request_error","param":"response_format.json_schema.schema","code":"missing_required_parameter"},"code":"missing_required_parameter","param":"response_format.json_schema.schema","type":"invalid_request_error"}
const completions = await generateFinalExtraction({
@@ -245,7 +245,7 @@ export async function extractController(
schema: req.body.schema,
});
- console.log("completions", completions);
+ // console.log("completions", completions);
// if(req.body.extract && req.body.formats.includes("extract")) {
// creditsToBeBilled = 5;
@@ -317,7 +317,7 @@ export async function extractController(
return res.status(200).json({
success: true,
- data: {}, // includeMetadata ? mapResults : linksToReturn,
+ data: completions.content, // includeMetadata ? mapResults : linksToReturn,
scrape_id: id, //origin?.includes("website") ? id : undefined,
});
}
\ No newline at end of file
diff --git a/apps/api/src/lib/extract/completions.ts b/apps/api/src/lib/extract/completions.ts
index fa75594bf..c02b3e315 100644
--- a/apps/api/src/lib/extract/completions.ts
+++ b/apps/api/src/lib/extract/completions.ts
@@ -77,6 +77,9 @@ export async function generateFinalExtraction({
required: ["items"],
additionalProperties: false,
};
+ } else if (schema) {
+ schema.additionalProperties = false;
+ schema.required = Object.keys(schema.properties);
}
const jsonCompletion = await openai.beta.chat.completions.parse({
@@ -96,7 +99,7 @@ export async function generateFinalExtraction({
type: "json_schema",
json_schema: {
name: "websiteContent",
- schema: schema.shape,
+ schema: schema,
strict: true,
},
}
diff --git a/apps/api/src/lib/ranker.ts b/apps/api/src/lib/ranker.ts
index 6c1646dad..7cd39820e 100644
--- a/apps/api/src/lib/ranker.ts
+++ b/apps/api/src/lib/ranker.ts
@@ -49,10 +49,10 @@ async function performRanking(links: string[], searchQuery: string) {
const linksAndScores = await Promise.all(links.map(async (link) => {
const linkEmbedding = await getEmbedding(link);
- console.log("linkEmbedding", linkEmbedding);
+ // console.log("linkEmbedding", linkEmbedding);
// const linkVector = textToVector(searchQuery, link);
const score = cosineSimilarity(queryEmbedding, linkEmbedding);
- console.log("score", score);
+ // console.log("score", score);
return { link, score };
}));