Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Refactor robustFetch logging to exclude sensitive parameters and improve error handling.
rafaelmmiller252a9ccc8994
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex e4c3802f1..d6bd51579 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -123,10 +123,11 @@ export async function robustFetch<return null as Output;}+ const resp = await request.text();response = {status: request.status,headers: request.headers,- body: await request.text(), // NOTE: can this throw an exception?+ body: resp, // NOTE: can this throw an exception?};} else {if (ignoreResponse === true) {@@ -171,7 +172,7 @@ export async function robustFetch<if (tryCount > 1) {logger.debug("Request sent failure status, trying " + (tryCount - 1) + " more times",- { params, response, requestId },+ { params: { ...params, logger: undefined }, response: { status: response.status, body: response.body }, requestId },);if (tryCooldown !== undefined) {await new Promise((resolve) =>@@ -186,14 +187,14 @@ export async function robustFetch<});} else {logger.debug("Request sent failure status", {- params,- response,+ params: { ...params, logger: undefined },+ response: { status: response.status, body: response.body },requestId,});throw new Error("Request sent failure status", {cause: {- params,- response,+ params: { ...params, logger: undefined },+ response: { status: response.status, body: response.body },requestId,},});@@ -217,13 +218,13 @@ export async function robustFetch<data = JSON.parse(response.body);} catch (error) {logger.debug("Request sent malformed JSON", {- params,- response,+ params: { ...params, logger: undefined },+ response: { status: response.status, body: response.body },requestId,});throw new Error("Request sent malformed JSON", {cause: {- params,+ params: { ...params, logger: undefined },response,requestId,},@@ -236,15 +237,15 @@ export async function robustFetch<} catch (error) {if (error instanceof ZodError) {logger.debug("Response does not match provided schema", {- params,- response,+ params: { ...params, logger: undefined },+ response: { status: response.status, body: response.body },requestId,error,schema,});throw new Error("Response does not match provided schema", {cause: {- params,+ params: { ...params, logger: undefined },response,requestId,error,@@ -253,15 +254,15 @@ export async function robustFetch<});} else {logger.debug("Parsing response with provided schema failed", {- params,- response,+ params: { ...params, logger: undefined },+ response: { status: response.status, body: response.body },requestId,error,schema,});throw new Error("Parsing response with provided schema failed", {cause: {- params,+ params: { ...params, logger: undefined },response,requestId,error,