Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixes
Nicolas67a29898742a
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 6d9b4a2c1..ededd63d4 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -23,6 +23,7 @@ import { getJobPriority } from "../../lib/job-priority";import { generateOpenAICompletions } from "../../scraper/scrapeURL/transformers/llmExtract";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { getMapResults } from "./map";+import { buildDocument } from "../../lib/extract/build-document";configDotenv();const redis = new Redis(process.env.REDIS_URL!);@@ -47,7 +48,7 @@ export async function extractController(if (url.includes('/*') || req.body.allowExternalLinks) {// Handle glob pattern URLsconst baseUrl = url.replace('/*', '');- const pathPrefix = baseUrl.split('/').slice(3).join('/'); // Get path after domain if any+ // const pathPrefix = baseUrl.split('/').slice(3).join('/'); // Get path after domain if anyconst allowExternalLinks = req.body.allowExternalLinks ?? true;let urlWithoutWww = baseUrl.replace("www.", "");@@ -76,9 +77,11 @@ export async function extractController(let mappedLinksRerank = mappedLinks.map(x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`);// Filter by path prefix if present- if (pathPrefix) {- mappedLinks = mappedLinks.filter(x => x.url && x.url.includes(`/${pathPrefix}/`));- }+ // console.log("pathPrefix", pathPrefix);+ // wrong+ // if (pathPrefix) {+ // mappedLinks = mappedLinks.filter(x => x.url && x.url.includes(`/${pathPrefix}/`));+ // }if (req.body.prompt) {const linksAndScores : { link: string, linkWithContext: string, score: number, originalIndex: number }[] = await performRanking(mappedLinksRerank, mappedLinks.map(l => l.url), mapUrl);@@ -170,6 +173,7 @@ export async function extractController(});}+ console.log("docs", docs.length);const completions = await generateOpenAICompletions(logger.child({ method: "extractController/generateOpenAICompletions" }),{@@ -178,7 +182,7 @@ export async function extractController(prompt: req.body.prompt,schema: req.body.schema,},- docs.map(x => x.markdown).join('\n')+ docs.map(x => buildDocument(x)).join('\n'));// console.log("completions", completions);diff --git a/apps/api/src/lib/extract/build-document.ts b/apps/api/src/lib/extract/build-document.tsnew file mode 100644index 000000000..7486d2a74--- /dev/null+++ b/apps/api/src/lib/extract/build-document.ts@@ -0,0 +1,17 @@+import { Document } from "../../controllers/v1/types";++export function buildDocument(document: Document): string {+ const metadata = document.metadata;+ const markdown = document.markdown;++ const documentMetadataString = `\nHere is the metadata for the document:\n${JSON.stringify(+ metadata,+ null,+ 2+ )}`;++ const documentString = `${markdown}${documentMetadataString}`;++ console.log("documentString", documentString);+ return markdown ?? "";+}diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex d8004c075..41d767791 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -63,7 +63,7 @@ export async function generateOpenAICompletions(logger: Logger, options: Extractlet warning: string | undefined;const openai = new OpenAI();- const model: TiktokenModel = (process.env.MODEL_NAME as TiktokenModel) ?? "gpt-4o";+ const model: TiktokenModel = (process.env.MODEL_NAME as TiktokenModel) ?? "gpt-4o-mini";if (markdown === undefined) {throw new Error("document.markdown is undefined -- this is unexpected");