Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): disable gcs logging without db auth (#2418)
delong3e9e6a8d78030
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 3e0ca1877..9a858e7c6 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -19,6 +19,11 @@ export async function logJob(force: boolean = false,bypassLogging: boolean = false,) {+ const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";+ if (!useDbAuthentication) {+ return;+ }+let logger = _logger.child({module: "log_job",method: "logJob",@@ -59,11 +64,6 @@ export async function logJob(});}- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";- if (!useDbAuthentication) {- return;- }-// Redact any pages that have an authorization header// actually, Don't. we use the db to retrieve results now. this breaks authed crawls - mogery// if (@@ -166,47 +166,68 @@ export async function logJob(job.url &&!zeroDataRetention) {- const hasMarkdown = hasFormatOfType(job.scrapeOptions?.formats, "markdown");- const hasChangeTracking = hasFormatOfType(job.scrapeOptions?.formats, "changeTracking");-+ const hasMarkdown = hasFormatOfType(+ job.scrapeOptions?.formats,+ "markdown",+ );+ const hasChangeTracking = hasFormatOfType(+ job.scrapeOptions?.formats,+ "changeTracking",+ );+// Insert if markdown format exists (either standalone or as part of change tracking)if (hasMarkdown || hasChangeTracking) {- await withSpan("firecrawl-log-job-change-tracking-insert", async span => {- setSpanAttributes(span, {- "log_job.operation": "change_tracking_insert",- "job.id": job.job_id,- "job.team_id": job.team_id,- "job.url": job.url,- "job.change_tracking_tag": job.change_tracking_tag ?? undefined,- });-- try {- const { error } = await supabase_service.rpc("change_tracking_insert_scrape", {- p_team_id: job.team_id,- p_url: job.url,- p_job_id: job.job_id,- p_change_tracking_tag: job.change_tracking_tag ?? null,- p_date_added: new Date().toISOString(),+ await withSpan(+ "firecrawl-log-job-change-tracking-insert",+ async span => {+ setSpanAttributes(span, {+ "log_job.operation": "change_tracking_insert",+ "job.id": job.job_id,+ "job.team_id": job.team_id,+ "job.url": job.url,+ "job.change_tracking_tag": job.change_tracking_tag ?? undefined,});- if (error) {- setSpanAttributes(span, { "change_tracking.insert_successful": false });- logger.warn("Error inserting into change_tracking_scrapes", {- error,- jobId: job.job_id,+ try {+ const { error } = await supabase_service.rpc(+ "change_tracking_insert_scrape",+ {+ p_team_id: job.team_id,+ p_url: job.url,+ p_job_id: job.job_id,+ p_change_tracking_tag: job.change_tracking_tag ?? null,+ p_date_added: new Date().toISOString(),+ },+ );++ if (error) {+ setSpanAttributes(span, {+ "change_tracking.insert_successful": false,+ });+ logger.warn("Error inserting into change_tracking_scrapes", {+ error,+ jobId: job.job_id,+ });+ } else {+ setSpanAttributes(span, {+ "change_tracking.insert_successful": true,+ });+ logger.debug("Change tracking record inserted successfully!");+ }+ } catch (error) {+ setSpanAttributes(span, {+ "change_tracking.insert_successful": false,});- } else {- setSpanAttributes(span, { "change_tracking.insert_successful": true });- logger.debug("Change tracking record inserted successfully!");+ logger.warn(+ "Exception while inserting into change_tracking_scrapes",+ {+ error,+ jobId: job.job_id,+ },+ );}- } catch (error) {- setSpanAttributes(span, { "change_tracking.insert_successful": false });- logger.warn("Exception while inserting into change_tracking_scrapes", {- error,- jobId: job.job_id,- });- }- });+ },+ );}}