Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Refactor robustFetch logging to exclude sensitive parameters and improve error handling.

rafaelmmiller252a9ccc8994
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index e4c3802f1..d6bd51579 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -123,10 +123,11 @@ export async function robustFetch<
return null as Output;
}
+ const resp = await request.text();
response = {
status: request.status,
headers: request.headers,
- body: await request.text(), // NOTE: can this throw an exception?
+ body: resp, // NOTE: can this throw an exception?
};
} else {
if (ignoreResponse === true) {
@@ -171,7 +172,7 @@ export async function robustFetch<
if (tryCount > 1) {
logger.debug(
"Request sent failure status, trying " + (tryCount - 1) + " more times",
- { params, response, requestId },
+ { params: { ...params, logger: undefined }, response: { status: response.status, body: response.body }, requestId },
);
if (tryCooldown !== undefined) {
await new Promise((resolve) =>
@@ -186,14 +187,14 @@ export async function robustFetch<
});
} else {
logger.debug("Request sent failure status", {
- params,
- response,
+ params: { ...params, logger: undefined },
+ response: { status: response.status, body: response.body },
requestId,
});
throw new Error("Request sent failure status", {
cause: {
- params,
- response,
+ params: { ...params, logger: undefined },
+ response: { status: response.status, body: response.body },
requestId,
},
});
@@ -217,13 +218,13 @@ export async function robustFetch<
data = JSON.parse(response.body);
} catch (error) {
logger.debug("Request sent malformed JSON", {
- params,
- response,
+ params: { ...params, logger: undefined },
+ response: { status: response.status, body: response.body },
requestId,
});
throw new Error("Request sent malformed JSON", {
cause: {
- params,
+ params: { ...params, logger: undefined },
response,
requestId,
},
@@ -236,15 +237,15 @@ export async function robustFetch<
} catch (error) {
if (error instanceof ZodError) {
logger.debug("Response does not match provided schema", {
- params,
- response,
+ params: { ...params, logger: undefined },
+ response: { status: response.status, body: response.body },
requestId,
error,
schema,
});
throw new Error("Response does not match provided schema", {
cause: {
- params,
+ params: { ...params, logger: undefined },
response,
requestId,
error,
@@ -253,15 +254,15 @@ export async function robustFetch<
});
} else {
logger.debug("Parsing response with provided schema failed", {
- params,
- response,
+ params: { ...params, logger: undefined },
+ response: { status: response.status, body: response.body },
requestId,
error,
schema,
});
throw new Error("Parsing response with provided schema failed", {
cause: {
- params,
+ params: { ...params, logger: undefined },
response,
requestId,
error,