Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update generate-llmstxt-service.ts

Nicolasd4cf2269ed6c
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
index 017ac669b..55c06124f 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
@@ -53,6 +53,7 @@ export async function performGenerateLlmsTxt(options: GenerateLLMsTextServiceOpt
showFullText: showFullText,
});
+
return {
success: true,
data: {
@@ -86,62 +87,72 @@ export async function performGenerateLlmsTxt(options: GenerateLLMsTextServiceOpt
let llmsFulltxt = `# ${url} llms-full.txt\n\n`;
- // Scrape each URL
- for (const url of urls) {
- _logger.debug(`Scraping URL: ${url}`);
- const document = await scrapeDocument(
- {
- url,
- teamId,
- plan,
- origin: url,
- timeout: 30000,
- isSingleUrl: true,
- },
- [],
- logger,
- { onlyMainContent: true }
- );
-
- if (!document) {
- logger.error(`Failed to scrape URL ${url}`);
- continue;
- }
-
- // Process scraped result
- if (!document.markdown) continue;
-
- _logger.debug(`Generating description for ${document.metadata?.url}`);
+ // Process URLs in batches of 10
+ for (let i = 0; i < urls.length; i += 10) {
+ const batch = urls.slice(i, i + 10);
- const completion = await openai.beta.chat.completions.parse({
- model: "gpt-4o-mini",
- messages: [
- {
- role: "user",
- content: `Generate a 9-10 word description and a 3-4 word title of the entire page based on ALL the content one will find on the page for this url: ${document.metadata?.url}. This will help in a user finding the page for its intended purpose. Here is the content: ${document.markdown}`
+ const batchResults = await Promise.all(batch.map(async (url) => {
+ _logger.debug(`Scraping URL: ${url}`);
+ try {
+ const document = await scrapeDocument(
+ {
+ url,
+ teamId,
+ plan,
+ origin: url,
+ timeout: 30000,
+ isSingleUrl: true,
+ },
+ [],
+ logger,
+ { onlyMainContent: true }
+ );
+
+ if (!document || !document.markdown) {
+ logger.error(`Failed to scrape URL ${url}`);
+ return null;
}
- ],
- response_format: zodResponseFormat(DescriptionSchema, "description")
- });
- try {
- const parsedResponse = completion.choices[0].message.parsed;
- const description = parsedResponse!.description;
- const title = parsedResponse!.title;
+ _logger.debug(`Generating description for ${document.metadata?.url}`);
+
+ const completion = await openai.beta.chat.completions.parse({
+ model: "gpt-4o-mini",
+ messages: [
+ {
+ role: "user",
+ content: `Generate a 9-10 word description and a 3-4 word title of the entire page based on ALL the content one will find on the page for this url: ${document.metadata?.url}. This will help in a user finding the page for its intended purpose. Here is the content: ${document.markdown}`
+ }
+ ],
+ response_format: zodResponseFormat(DescriptionSchema, "description")
+ });
+
+ const parsedResponse = completion.choices[0].message.parsed;
+ return {
+ title: parsedResponse!.title,
+ description: parsedResponse!.description,
+ url: document.metadata?.url,
+ markdown: document.markdown
+ };
+ } catch (error) {
+ logger.error(`Failed to process URL ${url}`, { error });
+ return null;
+ }
+ }));
+
+ // Process successful results from batch
+ for (const result of batchResults) {
+ if (!result) continue;
- llmstxt += `- [${title}](${document.metadata?.url}): ${description}\n`;
- llmsFulltxt += `## ${title}\n${document.markdown}\n\n`;
-
- // Update progress with both generated text and full text
- await updateGeneratedLlmsTxt(generationId, {
- status: "processing",
- generatedText: llmstxt,
- fullText: llmsFulltxt,
- });
- } catch (error) {
- logger.error(`Failed to parse LLM response for ${document.metadata?.url}`, { error });
- continue;
+ llmstxt += `- [${result.title}](${result.url}): ${result.description}\n`;
+ llmsFulltxt += `## ${result.title}\n${result.markdown}\n\n`;
}
+
+ // Update progress after each batch
+ await updateGeneratedLlmsTxt(generationId, {
+ status: "processing",
+ generatedText: llmstxt,
+ fullText: llmsFulltxt,
+ });
}
// After successful generation, save to cache