Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/gcs-jobs): consolidate GCS upload logic into a common function (#3606)
Gergő Móricz3d7a404779d3
diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.tsindex 0446a7168..623a8fad3 100644--- a/apps/api/src/lib/gcs-jobs.ts+++ b/apps/api/src/lib/gcs-jobs.ts@@ -11,20 +11,36 @@ import type {LoggedSearch,} from "../services/logging/log_job";import { config } from "../config";-import { validate, version } from "uuid";import crypto from "crypto";+import { Logger } from "winston";const credentials = config.GCS_CREDENTIALS? JSON.parse(atob(config.GCS_CREDENTIALS)): undefined;export const storage = new Storage({ credentials });+const storageManualRetries = new Storage({+ credentials,+ retryOptions: {+ autoRetry: false,+ maxRetries: 0,+ },+});++const BACKOFF_PARAMS = [0, 250, 1000];++type GCSOperationAttempt = {+ error: any;+ timeMs: number;+ backoffMs: number;+};+/*** Converts a job ID to a GCS filename.** Before the cutover, the filename is always `<id>.json`.* However, after we switched to v7 UUIDs, we realized that it's not working well with how GCS- * parititons GCS buckets, therefore, we need the filename to start with something random-esque+ * partitions GCS buckets, therefore, we need the filename to start with something random-esque* to smooth out the distribution of files between the partitions.* Therefore, after May 26, 2026, the filename is `<sha256(id)>-<id>.json`*@@ -32,7 +48,11 @@ export const storage = new Storage({ credentials });* @returns Filename for the job in GCS*/function idToFilename(id: string): string {- if (validate(id) && version(id) === 7) {+ if (+ id.match(+ /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-7[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/,+ )+ ) {const timestamp = parseInt(id.replace(/-/g, "").slice(0, 12), 16);const cutover = Date.UTC(2026, 4, 26, 0, 0, 0, 0); // Cutover at 2026-05-26 00:00:00 UTCif (timestamp < cutover) {@@ -45,15 +65,39 @@ function idToFilename(id: string): string {}}-export async function saveScrapeToGCS(scrape: LoggedScrape): Promise<void> {+async function saveJobToGCS(params: {+ mode: string;+ id: string;+ request_id: string;+ team_id: string;+ is_successful: boolean;+ num_docs: number;+ data: any;+ zeroDataRetention: boolean;+ metadata: any;+ logger: Logger;+}): Promise<void> {+ const filename = idToFilename(params.id);+ const logger = params.logger.child({+ module: "gcs-jobs",+ method: "saveJobToGCS",+ mode: params.mode,+ filename,+ zeroDataRetention: params.zeroDataRetention,+ });++ const saveAttempts: GCSOperationAttempt[] = [];+ const metadataAttempts: GCSOperationAttempt[] = [];+return await withSpan("firecrawl-gcs-save-job", async span => {setSpanAttributes(span, {"gcs.operation": "save_job",- "job.id": scrape.id,- "job.team_id": scrape.team_id,- "job.mode": "scrape",- "job.success": scrape.is_successful,- "job.num_docs": 1,+ "job.id": params.id,+ "job.request_id": params.request_id,+ "job.team_id": params.team_id,+ "job.mode": params.mode,+ "job.success": params.is_successful,+ "job.num_docs": params.num_docs,});if (!config.GCS_BUCKET_NAME) {@@ -61,455 +105,295 @@ export async function saveScrapeToGCS(scrape: LoggedScrape): Promise<void> {return;}- const bucket = storage.bucket(config.GCS_BUCKET_NAME);- const blob = bucket.file(idToFilename(scrape.id));+ const bucket = storageManualRetries.bucket(config.GCS_BUCKET_NAME);+ const blob = bucket.file(filename);// Save job docs with retry- for (let i = 0; i < 3; i++) {+ for (let i = 0; i < BACKOFF_PARAMS.length; i++) {+ const backoffMs = BACKOFF_PARAMS[i];+ if (backoffMs > 0) {+ await new Promise(resolve => setTimeout(resolve, backoffMs));+ }++ const saveStart = Date.now();try {- await blob.save(JSON.stringify([scrape.doc]), {+ await blob.save(JSON.stringify(params.data), {contentType: "application/json",});+ saveAttempts.push({+ error: null,+ timeMs: Date.now() - saveStart,+ backoffMs,+ });break;} catch (error) {- if (i === 2) {+ // TODO: determine what kind of errors we should backoff or instafail on+ saveAttempts.push({ error, timeMs: Date.now() - saveStart, backoffMs });++ if (i === BACKOFF_PARAMS.length - 1) {+ setSpanAttributes(span, { "gcs.save_successful": false });throw error;- } else {- logger.error(`Error saving job to GCS, retrying`, {- error,- scrapeId: scrape.id,- jobId: scrape.id,- i,- zeroDataRetention: scrape.zeroDataRetention,- });}}}// Save job metadata with retry- for (let i = 0; i < 3; i++) {+ for (let i = 0; i < BACKOFF_PARAMS.length; i++) {+ const backoffMs = BACKOFF_PARAMS[i];+ if (backoffMs > 0) {+ await new Promise(resolve => setTimeout(resolve, backoffMs));+ }++ const metadataStart = Date.now();try {await blob.setMetadata({- metadata: {- job_id: scrape.id ?? null,- success: scrape.is_successful,- message: scrape.zeroDataRetention ? null : (scrape.error ?? null),- num_docs: 1,- time_taken: scrape.time_taken,- team_id:- scrape.team_id === "preview" ||- scrape.team_id?.startsWith("preview_")- ? null- : scrape.team_id,- mode: "scrape",- url: scrape.zeroDataRetention- ? "<redacted due to zero data retention>"- : scrape.url,- page_options: scrape.zeroDataRetention- ? null- : JSON.stringify(scrape.options),- request_id: scrape.request_id ?? null,- },+ metadata: params.metadata,+ });+ metadataAttempts.push({+ error: null,+ timeMs: Date.now() - metadataStart,+ backoffMs,});break;} catch (error) {- if (i === 2) {+ // TODO: determine what kind of errors we should backoff or instafail on+ metadataAttempts.push({+ error,+ timeMs: Date.now() - metadataStart,+ backoffMs,+ });++ if (i === BACKOFF_PARAMS.length - 1) {+ setSpanAttributes(span, { "gcs.save_successful": false });throw error;- } else {- logger.error(`Error saving scrape metadata to GCS, retrying`, {- error,- scrapeId: scrape.id,- jobId: scrape.id,- i,- zeroDataRetention: scrape.zeroDataRetention,- });}}}setSpanAttributes(span, { "gcs.save_successful": true });- }).catch(error => {- logger.error(`Error saving scrape to GCS`, {- error,- scrapeId: scrape.id,- jobId: scrape.id,- zeroDataRetention: scrape.zeroDataRetention,- });- throw error;- });-}--export async function saveSearchToGCS(search: LoggedSearch): Promise<void> {- return await withSpan("firecrawl-gcs-save-search", async span => {- setSpanAttributes(span, {- "gcs.operation": "save_search",- "search.id": search.id,- "search.team_id": search.team_id,- request_id: search.request_id,- });-- if (!config.GCS_BUCKET_NAME) {- setSpanAttributes(span, { "gcs.bucket_configured": false });- return;- }-- const bucket = storage.bucket(config.GCS_BUCKET_NAME);- const blob = bucket.file(idToFilename(search.id));-- for (let i = 0; i < 3; i++) {- try {- await blob.save(JSON.stringify(search.results), {- contentType: "application/json",- });- setSpanAttributes(span, { "gcs.save_successful": true });- break;- } catch (error) {- if (i === 2) {- throw error;- } else {- logger.error(`Error saving search to GCS, retrying`, {- error,- searchId: search.id,- i,- });- }+ })+ .then(x => {+ if (saveAttempts.length === 0 && metadataAttempts.length === 0) {+ return x;}- }- for (let i = 0; i < 3; i++) {- try {- await blob.setMetadata({- metadata: {- mode: "search",- job_id: search.id,- num_docs: search.num_results,- time_taken: search.time_taken,- team_id:- search.team_id === "preview" ||- search.team_id?.startsWith("preview_")- ? null- : search.team_id,- query: search.zeroDataRetention- ? "<redacted due to zero data retention>"- : search.query,- options: search.zeroDataRetention- ? null- : JSON.stringify(search.options),- credits_cost: search.credits_cost,- success: search.is_successful,- error: search.zeroDataRetention ? null : (search.error ?? null),- num_results: search.num_results,- },+ if (saveAttempts.length === 1 && metadataAttempts.length === 1) {+ logger.debug("Job saved to GCS", {+ canonicalLog: "gcs-jobs/save",+ saveAttempts,+ metadataAttempts,+ success: true,});- setSpanAttributes(span, { "gcs.save_successful": true });- break;- } catch (error) {- if (i === 2) {- throw error;- } else {- logger.error(`Error saving search metadata to GCS, retrying`, {- error,- searchId: search.id,- i,- });- }- }- }- });-}--export async function saveExtractToGCS(extract: LoggedExtract): Promise<void> {- return await withSpan("firecrawl-gcs-save-extract", async span => {- setSpanAttributes(span, {- "gcs.operation": "save_extract",- "extract.id": extract.id,- "extract.team_id": extract.team_id,- });-- if (!config.GCS_BUCKET_NAME) {- setSpanAttributes(span, { "gcs.bucket_configured": false });- return;- }-- const bucket = storage.bucket(config.GCS_BUCKET_NAME);- const blob = bucket.file(idToFilename(extract.id));-- for (let i = 0; i < 3; i++) {- try {- await blob.save(JSON.stringify(extract.result), {- contentType: "application/json",+ } else {+ logger.warn("Job saved to GCS with retries", {+ canonicalLog: "gcs-jobs/save",+ saveAttempts,+ metadataAttempts,+ success: true,});- setSpanAttributes(span, { "gcs.save_successful": true });- break;- } catch (error) {- if (i === 2) {- throw error;- } else {- logger.error(`Error saving extract to GCS, retrying`, {- error,- extractId: extract.id,- i,- });- }}- }- for (let i = 0; i < 3; i++) {- try {- await blob.setMetadata({- metadata: {- mode: "extract",- job_id: extract.id,- num_docs: 1,- team_id:- extract.team_id === "preview" ||- extract.team_id?.startsWith("preview_")- ? null- : extract.team_id,- options: JSON.stringify(extract.options),- credits_cost: extract.credits_cost,- success: extract.is_successful,- error: extract.error ?? null,- },- });- setSpanAttributes(span, { "gcs.save_successful": true });- break;- } catch (error) {- if (i === 2) {- throw error;- } else {- logger.error(`Error saving extract metadata to GCS, retrying`, {- error,- extractId: extract.id,- i,- });- }- }- }+ return x;+ })+ .catch(error => {+ logger.error(`Job save to GCS failed`, {+ canonicalLog: "gcs-jobs/save",+ saveAttempts,+ metadataAttempts,+ success: false,+ error,+ });+ throw error;+ });+}- setSpanAttributes(span, { "gcs.save_successful": true });- return;+export async function saveScrapeToGCS(+ scrape: LoggedScrape,+ _logger: Logger = logger,+): Promise<void> {+ return await saveJobToGCS({+ mode: "scrape",+ id: scrape.id,+ team_id: scrape.team_id,+ is_successful: scrape.is_successful,+ request_id: scrape.request_id,+ num_docs: 1,+ data: [scrape.doc],+ zeroDataRetention: scrape.zeroDataRetention,+ logger: _logger,+ metadata: {+ job_id: scrape.id ?? null,+ success: scrape.is_successful,+ message: scrape.zeroDataRetention ? null : (scrape.error ?? null),+ num_docs: 1,+ time_taken: scrape.time_taken,+ team_id:+ scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")+ ? null+ : scrape.team_id,+ mode: "scrape",+ url: scrape.zeroDataRetention+ ? "<redacted due to zero data retention>"+ : scrape.url,+ page_options: scrape.zeroDataRetention+ ? null+ : JSON.stringify(scrape.options),+ request_id: scrape.request_id ?? null,+ },});}-export async function saveMapToGCS(map: LoggedMap): Promise<void> {- return await withSpan("firecrawl-gcs-save-map", async span => {- setSpanAttributes(span, {- "gcs.operation": "save_map",- "map.id": map.id,- "map.team_id": map.team_id,- });-- if (!config.GCS_BUCKET_NAME) {- setSpanAttributes(span, { "gcs.bucket_configured": false });- return;- }-- const bucket = storage.bucket(config.GCS_BUCKET_NAME);- const blob = bucket.file(idToFilename(map.id));-- for (let i = 0; i < 3; i++) {- try {- await blob.save(JSON.stringify(map.results), {- contentType: "application/json",- });- setSpanAttributes(span, { "gcs.save_successful": true });- break;- } catch (error) {- if (i === 2) {- throw error;- } else {- logger.error(`Error saving map to GCS, retrying`, {- error,- mapId: map.id,- i,- });- }- }- }+export async function saveSearchToGCS(+ search: LoggedSearch,+ _logger: Logger = logger,+): Promise<void> {+ return await saveJobToGCS({+ mode: "search",+ id: search.id,+ team_id: search.team_id,+ request_id: search.request_id,+ num_docs: search.num_results,+ data: search.results,+ metadata: {+ mode: "search",+ job_id: search.id,+ num_docs: search.num_results,+ time_taken: search.time_taken,+ team_id:+ search.team_id === "preview" || search.team_id?.startsWith("preview_")+ ? null+ : search.team_id,+ query: search.zeroDataRetention+ ? "<redacted due to zero data retention>"+ : search.query,+ options: search.zeroDataRetention ? null : JSON.stringify(search.options),+ credits_cost: search.credits_cost,+ success: search.is_successful,+ error: search.zeroDataRetention ? null : (search.error ?? null),+ num_results: search.num_results,+ },+ zeroDataRetention: search.zeroDataRetention,+ is_successful: search.is_successful,+ logger: _logger,+ });+}- for (let i = 0; i < 3; i++) {- try {- await blob.setMetadata({- metadata: {- mode: "map",- job_id: map.id,- num_results: map.results.length,- team_id:- map.team_id === "preview" || map.team_id?.startsWith("preview_")- ? null- : map.team_id,- options: JSON.stringify(map.options),- credits_cost: map.credits_cost,- success: true,- },- });- setSpanAttributes(span, { "gcs.save_successful": true });- break;- } catch (error) {- if (i === 2) {- throw error;- } else {- logger.error(`Error saving map metadata to GCS, retrying`, {- error,- mapId: map.id,- i,- });- }- }- }+export async function saveExtractToGCS(+ extract: LoggedExtract,+ _logger: Logger = logger,+): Promise<void> {+ return await saveJobToGCS({+ mode: "extract",+ id: extract.id,+ team_id: extract.team_id,+ request_id: extract.request_id,+ num_docs: 1,+ is_successful: extract.is_successful,+ data: extract.result,+ zeroDataRetention: false, // ZDR not supported on extract+ metadata: {+ mode: "extract",+ job_id: extract.id,+ num_docs: 1,+ team_id:+ extract.team_id === "preview" || extract.team_id?.startsWith("preview_")+ ? null+ : extract.team_id,+ options: JSON.stringify(extract.options),+ credits_cost: extract.credits_cost,+ success: extract.is_successful,+ error: extract.error ?? null,+ },+ logger: _logger,+ });+}- setSpanAttributes(span, { "gcs.save_successful": true });- return;+export async function saveMapToGCS(+ map: LoggedMap,+ _logger: Logger = logger,+): Promise<void> {+ return await saveJobToGCS({+ mode: "map",+ id: map.id,+ request_id: map.request_id,+ team_id: map.team_id,+ is_successful: true,+ num_docs: map.results.length,+ data: map.results,+ zeroDataRetention: map.zeroDataRetention,+ metadata: {+ mode: "map",+ job_id: map.id,+ num_results: map.results.length,+ team_id:+ map.team_id === "preview" || map.team_id?.startsWith("preview_")+ ? null+ : map.team_id,+ options: JSON.stringify(map.options),+ credits_cost: map.credits_cost,+ success: true,+ },+ logger: _logger,});}export async function saveDeepResearchToGCS(deepResearch: LoggedDeepResearch,+ _logger: Logger = logger,): Promise<void> {- return await withSpan("firecrawl-gcs-save-deep-research", async span => {- setSpanAttributes(span, {- "gcs.operation": "save_deep_research",- "deep_research.id": deepResearch.id,- "deep_research.team_id": deepResearch.team_id,- });-- if (!config.GCS_BUCKET_NAME) {- setSpanAttributes(span, { "gcs.bucket_configured": false });- return;- }-- const bucket = storage.bucket(config.GCS_BUCKET_NAME);- const blob = bucket.file(idToFilename(deepResearch.id));-- for (let i = 0; i < 3; i++) {- try {- await blob.save(JSON.stringify(deepResearch.result), {- contentType: "application/json",- });- setSpanAttributes(span, { "gcs.save_successful": true });- break;- } catch (error) {- if (i === 2) {- throw error;- } else {- logger.error(`Error saving deep research to GCS, retrying`, {- error,- deepResearchId: deepResearch.id,- i,- });- }- }- }-- for (let i = 0; i < 3; i++) {- try {- await blob.setMetadata({- metadata: {- mode: "deep_research",- job_id: deepResearch.id,- team_id:- deepResearch.team_id === "preview" ||- deepResearch.team_id?.startsWith("preview_")- ? null- : deepResearch.team_id,- options: JSON.stringify(deepResearch.options),- credits_cost: deepResearch.credits_cost,- success: true,- time_taken: deepResearch.time_taken,- },- });- setSpanAttributes(span, { "gcs.save_successful": true });- break;- } catch (error) {- if (i === 2) {- throw error;- } else {- logger.error(`Error saving deep research metadata to GCS, retrying`, {- error,- deepResearchId: deepResearch.id,- i,- });- }- }- }-- setSpanAttributes(span, { "gcs.save_successful": true });- return;+ return await saveJobToGCS({+ mode: "deep_research",+ id: deepResearch.id,+ request_id: deepResearch.request_id,+ team_id: deepResearch.team_id,+ is_successful: true,+ num_docs: 1,+ data: deepResearch.result,+ zeroDataRetention: false, // ZDR not supported on deep research+ metadata: {+ mode: "deep_research",+ job_id: deepResearch.id,+ team_id:+ deepResearch.team_id === "preview" ||+ deepResearch.team_id?.startsWith("preview_")+ ? null+ : deepResearch.team_id,+ options: JSON.stringify(deepResearch.options),+ credits_cost: deepResearch.credits_cost,+ success: true,+ time_taken: deepResearch.time_taken,+ },+ logger: _logger,});}-export async function saveLlmsTxtToGCS(llmsTxt: LoggedLlmsTxt): Promise<void> {- return await withSpan("firecrawl-gcs-save-llms-txt", async span => {- setSpanAttributes(span, {- "gcs.operation": "save_llms_txt",- "llms_txt.id": llmsTxt.id,- "llms_txt.team_id": llmsTxt.team_id,- });-- if (!config.GCS_BUCKET_NAME) {- setSpanAttributes(span, { "gcs.bucket_configured": false });- return;- }-- const bucket = storage.bucket(config.GCS_BUCKET_NAME);- const blob = bucket.file(idToFilename(llmsTxt.id));-- for (let i = 0; i < 3; i++) {- try {- await blob.save(JSON.stringify(llmsTxt.result), {- contentType: "application/json",- });- setSpanAttributes(span, { "gcs.save_successful": true });- break;- } catch (error) {- if (i === 2) {- throw error;- } else {- logger.error(`Error saving llms txt to GCS, retrying`, {- error,- llmsTxtId: llmsTxt.id,- i,- });- }- }- }-- for (let i = 0; i < 3; i++) {- try {- await blob.setMetadata({- metadata: {- mode: "llms_txt",- job_id: llmsTxt.id,- team_id:- llmsTxt.team_id === "preview" ||- llmsTxt.team_id?.startsWith("preview_")- ? null- : llmsTxt.team_id,- options: JSON.stringify(llmsTxt.options),- credits_cost: llmsTxt.credits_cost,- success: true,- num_urls: llmsTxt.num_urls,- cost_tracking: JSON.stringify(llmsTxt.cost_tracking),- },- });- setSpanAttributes(span, { "gcs.save_successful": true });- break;- } catch (error) {- if (i === 2) {- throw error;- } else {- logger.error(`Error saving llms txt metadata to GCS, retrying`, {- error,- llmsTxtId: llmsTxt.id,- i,- });- }- }- }-- setSpanAttributes(span, { "gcs.save_successful": true });- return;+export async function saveLlmsTxtToGCS(+ llmsTxt: LoggedLlmsTxt,+ _logger: Logger = logger,+): Promise<void> {+ return await saveJobToGCS({+ mode: "llms_txt",+ id: llmsTxt.id,+ team_id: llmsTxt.team_id,+ request_id: llmsTxt.request_id,+ num_docs: 1,+ is_successful: true,+ zeroDataRetention: false, // ZDR not supported on llms txt+ data: llmsTxt.result,+ metadata: {+ mode: "llms_txt",+ job_id: llmsTxt.id,+ team_id:+ llmsTxt.team_id === "preview" || llmsTxt.team_id?.startsWith("preview_")+ ? null+ : llmsTxt.team_id,+ options: JSON.stringify(llmsTxt.options),+ credits_cost: llmsTxt.credits_cost,+ success: true,+ num_urls: llmsTxt.num_urls,+ cost_tracking: JSON.stringify(llmsTxt.cost_tracking),+ },+ logger: _logger,});}diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 5241a6d37..bc5363303 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -278,7 +278,7 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {config.GCS_BUCKET_NAME &&!(scrape.skipNuq && scrape.zeroDataRetention)) {- await saveScrapeToGCS(scrape);+ await saveScrapeToGCS(scrape, logger);}if (@@ -461,7 +461,7 @@ export async function logSearch(search: LoggedSearch, force: boolean = false) {);if (search.results && !search.zeroDataRetention) {- await saveSearchToGCS(search);+ await saveSearchToGCS(search, logger);}}@@ -514,7 +514,7 @@ export async function logExtract(if (extract.result) {if (config.GCS_BUCKET_NAME) {- await saveExtractToGCS(extract);+ await saveExtractToGCS(extract, logger);} else {// Fallback: save result to Redis with 24h TTL when GCS is not configuredawait saveExtractResult(extract.id, extract.result);@@ -564,7 +564,7 @@ export async function logMap(map: LoggedMap, force: boolean = false) {);if (map.results && !map.zeroDataRetention) {- await saveMapToGCS(map);+ await saveMapToGCS(map, logger);}}@@ -612,7 +612,7 @@ export async function logLlmsTxt();if (llmsTxt.result) {- await saveLlmsTxtToGCS(llmsTxt);+ await saveLlmsTxtToGCS(llmsTxt, logger);}}@@ -661,6 +661,6 @@ export async function logDeepResearch();if (deepResearch.result) {- await saveDeepResearchToGCS(deepResearch);+ await saveDeepResearchToGCS(deepResearch, logger);}}