Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix types

Gergő Móriczf5abba801497
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index 31d58eff6..dec7485e6 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -22,6 +22,7 @@ import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
import { CostTracking } from "../../lib/extract/extraction-service";
import { supabase_service } from "../../services/supabase";
+import { fromV1ScrapeOptions } from "../v2/types";
interface DocumentWithCostTracking {
document: Document;
@@ -101,16 +102,17 @@ async function scrapeX402SearchResult(
origin: options.origin,
zeroDataRetention,
});
+ const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(options.scrapeOptions, options.timeout, options.teamId);
await addScrapeJob(
{
url: searchResult.url,
mode: "single_urls" as Mode,
team_id: options.teamId,
scrapeOptions: {
- ...options.scrapeOptions,
- maxAge: 3 * 24 * 60 * 60 * 1000, // 3 days
+ ...scrapeOptions,
+ maxAge: scrapeOptions.maxAge === 0 ? 3 * 24 * 60 * 60 * 1000 : scrapeOptions.maxAge,
},
- internalOptions: { teamId: options.teamId, bypassBilling: true, zeroDataRetention },
+ internalOptions: { ...internalOptions, teamId: options.teamId, bypassBilling: true, zeroDataRetention },
origin: options.origin,
is_scrape: true,
startTime: Date.now(),
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.ts
index b951784e9..4199f910e 100644
--- a/apps/api/src/controllers/v2/crawl-params-preview.ts
+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts
@@ -61,7 +61,8 @@ export async function crawlParamsPreviewController(
const { extract } = await generateCrawlerOptionsFromPrompt(
parsedBody.prompt,
logger,
- costTracking
+ costTracking,
+ { teamId: req.auth.team_id }
);
const generatedOptions = extract || {};
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 04c4a798d..10b8922dc 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -68,7 +68,8 @@ export async function crawlController(
const { extract } = await generateCrawlerOptionsFromPrompt(
req.body.prompt,
logger,
- costTracking
+ costTracking,
+ { teamId: req.auth.team_id, crawlId: id }
);
promptGeneratedOptions = extract || {};
logger.debug("Generated crawler options from prompt", {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 3fe3ffc8d..b20250320 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -6,13 +6,11 @@ import { EngineError, IndexMissError, TimeoutError } from "../../error";
import crypto from "crypto";
export async function sendDocumentToIndex(meta: Meta, document: Document) {
-
-
const shouldCache = meta.options.storeInCache
&& !meta.internalOptions.zeroDataRetention
&& meta.winnerEngine !== "index"
&& meta.winnerEngine !== "index;documents"
- && !(meta.winnerEngine === "pdf" && meta.options.parsePDF === false)
+ && !(meta.winnerEngine === "pdf" && meta.options.parsers?.includes("pdf") === false)
&& (
meta.internalOptions.teamId === "sitemap"
|| (
@@ -212,13 +210,13 @@ export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefin
const isCachedPdfBase64 = doc.html && doc.html.startsWith("JVBERi");
// If the cached content is base64 PDF but we want parsed PDF (parsePDF:true or default)
- if (isCachedPdfBase64 && meta.options.parsePDF !== false) {
+ if (isCachedPdfBase64 && meta.options.parsers?.includes("pdf") !== false) {
// Cached content is unparsed PDF, but we want parsed - report cache miss
throw new IndexMissError();
}
// If the cached content is NOT base64 PDF but we want unparsed PDF (parsePDF:false)
- if (!isCachedPdfBase64 && meta.options.parsePDF === false) {
+ if (!isCachedPdfBase64 && meta.options.parsers?.includes("pdf") === false) {
// Check if URL looks like a PDF
const isPdfUrl = meta.url.toLowerCase().endsWith(".pdf") || meta.url.includes(".pdf?");
if (isPdfUrl) {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 34b1252f4..25088db8c 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -612,8 +612,8 @@ export async function scrapeURL(
&& abRate > 0
&& Math.random() <= abRate
&& abHostEnv
- && meta.options.agent === undefined
- && (meta.options.extract || meta.options.jsonOptions)?.agent === undefined;
+ && meta.internalOptions.v1Agent === undefined
+ && meta.internalOptions.v1JSONAgent === undefined;
if (shouldABTest) {
(async () => {
try {
@@ -621,7 +621,7 @@ export async function scrapeURL(
abLogger.info("A/B-testing scrapeURL to staging");
const abort = AbortSignal.timeout(Math.min(60000, (meta.options.timeout ?? 30000) + 10000));
await robustFetch({
- url: `http://${abHostEnv}/v1/scrape`,
+ url: `http://${abHostEnv}/v2/scrape`,
method: "POST",
body: {
url: meta.url,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 71a1215a3..98ee0ffde 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -957,6 +957,11 @@ export async function performSummary(
method: "performSummary",
},
},
+ metadata: {
+ teamId: meta.internalOptions.teamId,
+ functionId: "performSummary",
+ scrapeId: meta.id,
+ }
};
const {
@@ -1109,6 +1114,7 @@ export async function generateCrawlerOptionsFromPrompt(
prompt: string,
logger: Logger,
costTracking: CostTracking,
+ metadata: { teamId: string, crawlId?: string },
): Promise<{ extract: any }> {
const model = getModel("gpt-4o", "openai");
const retryModel = getModel("gpt-4o-mini", "openai");
@@ -1151,6 +1157,10 @@ Return a JSON object with only the relevant options for the user's request. Don'
method: "generateCrawlerOptionsFromPrompt",
},
},
+ metadata: {
+ ...metadata,
+ functionId: "generateCrawlerOptionsFromPrompt",
+ }
});
return { extract };