Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): disable gcs logging without db auth (#2418)

delong3e9e6a8d78030
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 3e0ca1877..9a858e7c6 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -19,6 +19,11 @@ export async function logJob(
force: boolean = false,
bypassLogging: boolean = false,
) {
+ const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
+ if (!useDbAuthentication) {
+ return;
+ }
+
let logger = _logger.child({
module: "log_job",
method: "logJob",
@@ -59,11 +64,6 @@ export async function logJob(
});
}
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
- if (!useDbAuthentication) {
- return;
- }
-
// Redact any pages that have an authorization header
// actually, Don't. we use the db to retrieve results now. this breaks authed crawls - mogery
// if (
@@ -166,47 +166,68 @@ export async function logJob(
job.url &&
!zeroDataRetention
) {
- const hasMarkdown = hasFormatOfType(job.scrapeOptions?.formats, "markdown");
- const hasChangeTracking = hasFormatOfType(job.scrapeOptions?.formats, "changeTracking");
-
+ const hasMarkdown = hasFormatOfType(
+ job.scrapeOptions?.formats,
+ "markdown",
+ );
+ const hasChangeTracking = hasFormatOfType(
+ job.scrapeOptions?.formats,
+ "changeTracking",
+ );
+
// Insert if markdown format exists (either standalone or as part of change tracking)
if (hasMarkdown || hasChangeTracking) {
- await withSpan("firecrawl-log-job-change-tracking-insert", async span => {
- setSpanAttributes(span, {
- "log_job.operation": "change_tracking_insert",
- "job.id": job.job_id,
- "job.team_id": job.team_id,
- "job.url": job.url,
- "job.change_tracking_tag": job.change_tracking_tag ?? undefined,
- });
-
- try {
- const { error } = await supabase_service.rpc("change_tracking_insert_scrape", {
- p_team_id: job.team_id,
- p_url: job.url,
- p_job_id: job.job_id,
- p_change_tracking_tag: job.change_tracking_tag ?? null,
- p_date_added: new Date().toISOString(),
+ await withSpan(
+ "firecrawl-log-job-change-tracking-insert",
+ async span => {
+ setSpanAttributes(span, {
+ "log_job.operation": "change_tracking_insert",
+ "job.id": job.job_id,
+ "job.team_id": job.team_id,
+ "job.url": job.url,
+ "job.change_tracking_tag": job.change_tracking_tag ?? undefined,
});
- if (error) {
- setSpanAttributes(span, { "change_tracking.insert_successful": false });
- logger.warn("Error inserting into change_tracking_scrapes", {
- error,
- jobId: job.job_id,
+ try {
+ const { error } = await supabase_service.rpc(
+ "change_tracking_insert_scrape",
+ {
+ p_team_id: job.team_id,
+ p_url: job.url,
+ p_job_id: job.job_id,
+ p_change_tracking_tag: job.change_tracking_tag ?? null,
+ p_date_added: new Date().toISOString(),
+ },
+ );
+
+ if (error) {
+ setSpanAttributes(span, {
+ "change_tracking.insert_successful": false,
+ });
+ logger.warn("Error inserting into change_tracking_scrapes", {
+ error,
+ jobId: job.job_id,
+ });
+ } else {
+ setSpanAttributes(span, {
+ "change_tracking.insert_successful": true,
+ });
+ logger.debug("Change tracking record inserted successfully!");
+ }
+ } catch (error) {
+ setSpanAttributes(span, {
+ "change_tracking.insert_successful": false,
});
- } else {
- setSpanAttributes(span, { "change_tracking.insert_successful": true });
- logger.debug("Change tracking record inserted successfully!");
+ logger.warn(
+ "Exception while inserting into change_tracking_scrapes",
+ {
+ error,
+ jobId: job.job_id,
+ },
+ );
}
- } catch (error) {
- setSpanAttributes(span, { "change_tracking.insert_successful": false });
- logger.warn("Exception while inserting into change_tracking_scrapes", {
- error,
- jobId: job.job_id,
- });
- }
- });
+ },
+ );
}
}