Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/gcs-jobs): consolidate GCS upload logic into a common function (#3606)

Gergő Móricz3d7a404779d3
diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.ts
index 0446a7168..623a8fad3 100644
--- a/apps/api/src/lib/gcs-jobs.ts
+++ b/apps/api/src/lib/gcs-jobs.ts
@@ -11,20 +11,36 @@ import type {
LoggedSearch,
} from "../services/logging/log_job";
import { config } from "../config";
-import { validate, version } from "uuid";
import crypto from "crypto";
+import { Logger } from "winston";
const credentials = config.GCS_CREDENTIALS
? JSON.parse(atob(config.GCS_CREDENTIALS))
: undefined;
export const storage = new Storage({ credentials });
+const storageManualRetries = new Storage({
+ credentials,
+ retryOptions: {
+ autoRetry: false,
+ maxRetries: 0,
+ },
+});
+
+const BACKOFF_PARAMS = [0, 250, 1000];
+
+type GCSOperationAttempt = {
+ error: any;
+ timeMs: number;
+ backoffMs: number;
+};
+
/**
* Converts a job ID to a GCS filename.
*
* Before the cutover, the filename is always `<id>.json`.
* However, after we switched to v7 UUIDs, we realized that it's not working well with how GCS
- * parititons GCS buckets, therefore, we need the filename to start with something random-esque
+ * partitions GCS buckets, therefore, we need the filename to start with something random-esque
* to smooth out the distribution of files between the partitions.
* Therefore, after May 26, 2026, the filename is `<sha256(id)>-<id>.json`
*
@@ -32,7 +48,11 @@ export const storage = new Storage({ credentials });
* @returns Filename for the job in GCS
*/
function idToFilename(id: string): string {
- if (validate(id) && version(id) === 7) {
+ if (
+ id.match(
+ /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-7[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/,
+ )
+ ) {
const timestamp = parseInt(id.replace(/-/g, "").slice(0, 12), 16);
const cutover = Date.UTC(2026, 4, 26, 0, 0, 0, 0); // Cutover at 2026-05-26 00:00:00 UTC
if (timestamp < cutover) {
@@ -45,15 +65,39 @@ function idToFilename(id: string): string {
}
}
-export async function saveScrapeToGCS(scrape: LoggedScrape): Promise<void> {
+async function saveJobToGCS(params: {
+ mode: string;
+ id: string;
+ request_id: string;
+ team_id: string;
+ is_successful: boolean;
+ num_docs: number;
+ data: any;
+ zeroDataRetention: boolean;
+ metadata: any;
+ logger: Logger;
+}): Promise<void> {
+ const filename = idToFilename(params.id);
+ const logger = params.logger.child({
+ module: "gcs-jobs",
+ method: "saveJobToGCS",
+ mode: params.mode,
+ filename,
+ zeroDataRetention: params.zeroDataRetention,
+ });
+
+ const saveAttempts: GCSOperationAttempt[] = [];
+ const metadataAttempts: GCSOperationAttempt[] = [];
+
return await withSpan("firecrawl-gcs-save-job", async span => {
setSpanAttributes(span, {
"gcs.operation": "save_job",
- "job.id": scrape.id,
- "job.team_id": scrape.team_id,
- "job.mode": "scrape",
- "job.success": scrape.is_successful,
- "job.num_docs": 1,
+ "job.id": params.id,
+ "job.request_id": params.request_id,
+ "job.team_id": params.team_id,
+ "job.mode": params.mode,
+ "job.success": params.is_successful,
+ "job.num_docs": params.num_docs,
});
if (!config.GCS_BUCKET_NAME) {
@@ -61,455 +105,295 @@ export async function saveScrapeToGCS(scrape: LoggedScrape): Promise<void> {
return;
}
- const bucket = storage.bucket(config.GCS_BUCKET_NAME);
- const blob = bucket.file(idToFilename(scrape.id));
+ const bucket = storageManualRetries.bucket(config.GCS_BUCKET_NAME);
+ const blob = bucket.file(filename);
// Save job docs with retry
- for (let i = 0; i < 3; i++) {
+ for (let i = 0; i < BACKOFF_PARAMS.length; i++) {
+ const backoffMs = BACKOFF_PARAMS[i];
+ if (backoffMs > 0) {
+ await new Promise(resolve => setTimeout(resolve, backoffMs));
+ }
+
+ const saveStart = Date.now();
try {
- await blob.save(JSON.stringify([scrape.doc]), {
+ await blob.save(JSON.stringify(params.data), {
contentType: "application/json",
});
+ saveAttempts.push({
+ error: null,
+ timeMs: Date.now() - saveStart,
+ backoffMs,
+ });
break;
} catch (error) {
- if (i === 2) {
+ // TODO: determine what kind of errors we should backoff or instafail on
+ saveAttempts.push({ error, timeMs: Date.now() - saveStart, backoffMs });
+
+ if (i === BACKOFF_PARAMS.length - 1) {
+ setSpanAttributes(span, { "gcs.save_successful": false });
throw error;
- } else {
- logger.error(`Error saving job to GCS, retrying`, {
- error,
- scrapeId: scrape.id,
- jobId: scrape.id,
- i,
- zeroDataRetention: scrape.zeroDataRetention,
- });
}
}
}
// Save job metadata with retry
- for (let i = 0; i < 3; i++) {
+ for (let i = 0; i < BACKOFF_PARAMS.length; i++) {
+ const backoffMs = BACKOFF_PARAMS[i];
+ if (backoffMs > 0) {
+ await new Promise(resolve => setTimeout(resolve, backoffMs));
+ }
+
+ const metadataStart = Date.now();
try {
await blob.setMetadata({
- metadata: {
- job_id: scrape.id ?? null,
- success: scrape.is_successful,
- message: scrape.zeroDataRetention ? null : (scrape.error ?? null),
- num_docs: 1,
- time_taken: scrape.time_taken,
- team_id:
- scrape.team_id === "preview" ||
- scrape.team_id?.startsWith("preview_")
- ? null
- : scrape.team_id,
- mode: "scrape",
- url: scrape.zeroDataRetention
- ? "<redacted due to zero data retention>"
- : scrape.url,
- page_options: scrape.zeroDataRetention
- ? null
- : JSON.stringify(scrape.options),
- request_id: scrape.request_id ?? null,
- },
+ metadata: params.metadata,
+ });
+ metadataAttempts.push({
+ error: null,
+ timeMs: Date.now() - metadataStart,
+ backoffMs,
});
break;
} catch (error) {
- if (i === 2) {
+ // TODO: determine what kind of errors we should backoff or instafail on
+ metadataAttempts.push({
+ error,
+ timeMs: Date.now() - metadataStart,
+ backoffMs,
+ });
+
+ if (i === BACKOFF_PARAMS.length - 1) {
+ setSpanAttributes(span, { "gcs.save_successful": false });
throw error;
- } else {
- logger.error(`Error saving scrape metadata to GCS, retrying`, {
- error,
- scrapeId: scrape.id,
- jobId: scrape.id,
- i,
- zeroDataRetention: scrape.zeroDataRetention,
- });
}
}
}
setSpanAttributes(span, { "gcs.save_successful": true });
- }).catch(error => {
- logger.error(`Error saving scrape to GCS`, {
- error,
- scrapeId: scrape.id,
- jobId: scrape.id,
- zeroDataRetention: scrape.zeroDataRetention,
- });
- throw error;
- });
-}
-
-export async function saveSearchToGCS(search: LoggedSearch): Promise<void> {
- return await withSpan("firecrawl-gcs-save-search", async span => {
- setSpanAttributes(span, {
- "gcs.operation": "save_search",
- "search.id": search.id,
- "search.team_id": search.team_id,
- request_id: search.request_id,
- });
-
- if (!config.GCS_BUCKET_NAME) {
- setSpanAttributes(span, { "gcs.bucket_configured": false });
- return;
- }
-
- const bucket = storage.bucket(config.GCS_BUCKET_NAME);
- const blob = bucket.file(idToFilename(search.id));
-
- for (let i = 0; i < 3; i++) {
- try {
- await blob.save(JSON.stringify(search.results), {
- contentType: "application/json",
- });
- setSpanAttributes(span, { "gcs.save_successful": true });
- break;
- } catch (error) {
- if (i === 2) {
- throw error;
- } else {
- logger.error(`Error saving search to GCS, retrying`, {
- error,
- searchId: search.id,
- i,
- });
- }
+ })
+ .then(x => {
+ if (saveAttempts.length === 0 && metadataAttempts.length === 0) {
+ return x;
}
- }
- for (let i = 0; i < 3; i++) {
- try {
- await blob.setMetadata({
- metadata: {
- mode: "search",
- job_id: search.id,
- num_docs: search.num_results,
- time_taken: search.time_taken,
- team_id:
- search.team_id === "preview" ||
- search.team_id?.startsWith("preview_")
- ? null
- : search.team_id,
- query: search.zeroDataRetention
- ? "<redacted due to zero data retention>"
- : search.query,
- options: search.zeroDataRetention
- ? null
- : JSON.stringify(search.options),
- credits_cost: search.credits_cost,
- success: search.is_successful,
- error: search.zeroDataRetention ? null : (search.error ?? null),
- num_results: search.num_results,
- },
+ if (saveAttempts.length === 1 && metadataAttempts.length === 1) {
+ logger.debug("Job saved to GCS", {
+ canonicalLog: "gcs-jobs/save",
+ saveAttempts,
+ metadataAttempts,
+ success: true,
});
- setSpanAttributes(span, { "gcs.save_successful": true });
- break;
- } catch (error) {
- if (i === 2) {
- throw error;
- } else {
- logger.error(`Error saving search metadata to GCS, retrying`, {
- error,
- searchId: search.id,
- i,
- });
- }
- }
- }
- });
-}
-
-export async function saveExtractToGCS(extract: LoggedExtract): Promise<void> {
- return await withSpan("firecrawl-gcs-save-extract", async span => {
- setSpanAttributes(span, {
- "gcs.operation": "save_extract",
- "extract.id": extract.id,
- "extract.team_id": extract.team_id,
- });
-
- if (!config.GCS_BUCKET_NAME) {
- setSpanAttributes(span, { "gcs.bucket_configured": false });
- return;
- }
-
- const bucket = storage.bucket(config.GCS_BUCKET_NAME);
- const blob = bucket.file(idToFilename(extract.id));
-
- for (let i = 0; i < 3; i++) {
- try {
- await blob.save(JSON.stringify(extract.result), {
- contentType: "application/json",
+ } else {
+ logger.warn("Job saved to GCS with retries", {
+ canonicalLog: "gcs-jobs/save",
+ saveAttempts,
+ metadataAttempts,
+ success: true,
});
- setSpanAttributes(span, { "gcs.save_successful": true });
- break;
- } catch (error) {
- if (i === 2) {
- throw error;
- } else {
- logger.error(`Error saving extract to GCS, retrying`, {
- error,
- extractId: extract.id,
- i,
- });
- }
}
- }
- for (let i = 0; i < 3; i++) {
- try {
- await blob.setMetadata({
- metadata: {
- mode: "extract",
- job_id: extract.id,
- num_docs: 1,
- team_id:
- extract.team_id === "preview" ||
- extract.team_id?.startsWith("preview_")
- ? null
- : extract.team_id,
- options: JSON.stringify(extract.options),
- credits_cost: extract.credits_cost,
- success: extract.is_successful,
- error: extract.error ?? null,
- },
- });
- setSpanAttributes(span, { "gcs.save_successful": true });
- break;
- } catch (error) {
- if (i === 2) {
- throw error;
- } else {
- logger.error(`Error saving extract metadata to GCS, retrying`, {
- error,
- extractId: extract.id,
- i,
- });
- }
- }
- }
+ return x;
+ })
+ .catch(error => {
+ logger.error(`Job save to GCS failed`, {
+ canonicalLog: "gcs-jobs/save",
+ saveAttempts,
+ metadataAttempts,
+ success: false,
+ error,
+ });
+ throw error;
+ });
+}
- setSpanAttributes(span, { "gcs.save_successful": true });
- return;
+export async function saveScrapeToGCS(
+ scrape: LoggedScrape,
+ _logger: Logger = logger,
+): Promise<void> {
+ return await saveJobToGCS({
+ mode: "scrape",
+ id: scrape.id,
+ team_id: scrape.team_id,
+ is_successful: scrape.is_successful,
+ request_id: scrape.request_id,
+ num_docs: 1,
+ data: [scrape.doc],
+ zeroDataRetention: scrape.zeroDataRetention,
+ logger: _logger,
+ metadata: {
+ job_id: scrape.id ?? null,
+ success: scrape.is_successful,
+ message: scrape.zeroDataRetention ? null : (scrape.error ?? null),
+ num_docs: 1,
+ time_taken: scrape.time_taken,
+ team_id:
+ scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")
+ ? null
+ : scrape.team_id,
+ mode: "scrape",
+ url: scrape.zeroDataRetention
+ ? "<redacted due to zero data retention>"
+ : scrape.url,
+ page_options: scrape.zeroDataRetention
+ ? null
+ : JSON.stringify(scrape.options),
+ request_id: scrape.request_id ?? null,
+ },
});
}
-export async function saveMapToGCS(map: LoggedMap): Promise<void> {
- return await withSpan("firecrawl-gcs-save-map", async span => {
- setSpanAttributes(span, {
- "gcs.operation": "save_map",
- "map.id": map.id,
- "map.team_id": map.team_id,
- });
-
- if (!config.GCS_BUCKET_NAME) {
- setSpanAttributes(span, { "gcs.bucket_configured": false });
- return;
- }
-
- const bucket = storage.bucket(config.GCS_BUCKET_NAME);
- const blob = bucket.file(idToFilename(map.id));
-
- for (let i = 0; i < 3; i++) {
- try {
- await blob.save(JSON.stringify(map.results), {
- contentType: "application/json",
- });
- setSpanAttributes(span, { "gcs.save_successful": true });
- break;
- } catch (error) {
- if (i === 2) {
- throw error;
- } else {
- logger.error(`Error saving map to GCS, retrying`, {
- error,
- mapId: map.id,
- i,
- });
- }
- }
- }
+export async function saveSearchToGCS(
+ search: LoggedSearch,
+ _logger: Logger = logger,
+): Promise<void> {
+ return await saveJobToGCS({
+ mode: "search",
+ id: search.id,
+ team_id: search.team_id,
+ request_id: search.request_id,
+ num_docs: search.num_results,
+ data: search.results,
+ metadata: {
+ mode: "search",
+ job_id: search.id,
+ num_docs: search.num_results,
+ time_taken: search.time_taken,
+ team_id:
+ search.team_id === "preview" || search.team_id?.startsWith("preview_")
+ ? null
+ : search.team_id,
+ query: search.zeroDataRetention
+ ? "<redacted due to zero data retention>"
+ : search.query,
+ options: search.zeroDataRetention ? null : JSON.stringify(search.options),
+ credits_cost: search.credits_cost,
+ success: search.is_successful,
+ error: search.zeroDataRetention ? null : (search.error ?? null),
+ num_results: search.num_results,
+ },
+ zeroDataRetention: search.zeroDataRetention,
+ is_successful: search.is_successful,
+ logger: _logger,
+ });
+}
- for (let i = 0; i < 3; i++) {
- try {
- await blob.setMetadata({
- metadata: {
- mode: "map",
- job_id: map.id,
- num_results: map.results.length,
- team_id:
- map.team_id === "preview" || map.team_id?.startsWith("preview_")
- ? null
- : map.team_id,
- options: JSON.stringify(map.options),
- credits_cost: map.credits_cost,
- success: true,
- },
- });
- setSpanAttributes(span, { "gcs.save_successful": true });
- break;
- } catch (error) {
- if (i === 2) {
- throw error;
- } else {
- logger.error(`Error saving map metadata to GCS, retrying`, {
- error,
- mapId: map.id,
- i,
- });
- }
- }
- }
+export async function saveExtractToGCS(
+ extract: LoggedExtract,
+ _logger: Logger = logger,
+): Promise<void> {
+ return await saveJobToGCS({
+ mode: "extract",
+ id: extract.id,
+ team_id: extract.team_id,
+ request_id: extract.request_id,
+ num_docs: 1,
+ is_successful: extract.is_successful,
+ data: extract.result,
+ zeroDataRetention: false, // ZDR not supported on extract
+ metadata: {
+ mode: "extract",
+ job_id: extract.id,
+ num_docs: 1,
+ team_id:
+ extract.team_id === "preview" || extract.team_id?.startsWith("preview_")
+ ? null
+ : extract.team_id,
+ options: JSON.stringify(extract.options),
+ credits_cost: extract.credits_cost,
+ success: extract.is_successful,
+ error: extract.error ?? null,
+ },
+ logger: _logger,
+ });
+}
- setSpanAttributes(span, { "gcs.save_successful": true });
- return;
+export async function saveMapToGCS(
+ map: LoggedMap,
+ _logger: Logger = logger,
+): Promise<void> {
+ return await saveJobToGCS({
+ mode: "map",
+ id: map.id,
+ request_id: map.request_id,
+ team_id: map.team_id,
+ is_successful: true,
+ num_docs: map.results.length,
+ data: map.results,
+ zeroDataRetention: map.zeroDataRetention,
+ metadata: {
+ mode: "map",
+ job_id: map.id,
+ num_results: map.results.length,
+ team_id:
+ map.team_id === "preview" || map.team_id?.startsWith("preview_")
+ ? null
+ : map.team_id,
+ options: JSON.stringify(map.options),
+ credits_cost: map.credits_cost,
+ success: true,
+ },
+ logger: _logger,
});
}
export async function saveDeepResearchToGCS(
deepResearch: LoggedDeepResearch,
+ _logger: Logger = logger,
): Promise<void> {
- return await withSpan("firecrawl-gcs-save-deep-research", async span => {
- setSpanAttributes(span, {
- "gcs.operation": "save_deep_research",
- "deep_research.id": deepResearch.id,
- "deep_research.team_id": deepResearch.team_id,
- });
-
- if (!config.GCS_BUCKET_NAME) {
- setSpanAttributes(span, { "gcs.bucket_configured": false });
- return;
- }
-
- const bucket = storage.bucket(config.GCS_BUCKET_NAME);
- const blob = bucket.file(idToFilename(deepResearch.id));
-
- for (let i = 0; i < 3; i++) {
- try {
- await blob.save(JSON.stringify(deepResearch.result), {
- contentType: "application/json",
- });
- setSpanAttributes(span, { "gcs.save_successful": true });
- break;
- } catch (error) {
- if (i === 2) {
- throw error;
- } else {
- logger.error(`Error saving deep research to GCS, retrying`, {
- error,
- deepResearchId: deepResearch.id,
- i,
- });
- }
- }
- }
-
- for (let i = 0; i < 3; i++) {
- try {
- await blob.setMetadata({
- metadata: {
- mode: "deep_research",
- job_id: deepResearch.id,
- team_id:
- deepResearch.team_id === "preview" ||
- deepResearch.team_id?.startsWith("preview_")
- ? null
- : deepResearch.team_id,
- options: JSON.stringify(deepResearch.options),
- credits_cost: deepResearch.credits_cost,
- success: true,
- time_taken: deepResearch.time_taken,
- },
- });
- setSpanAttributes(span, { "gcs.save_successful": true });
- break;
- } catch (error) {
- if (i === 2) {
- throw error;
- } else {
- logger.error(`Error saving deep research metadata to GCS, retrying`, {
- error,
- deepResearchId: deepResearch.id,
- i,
- });
- }
- }
- }
-
- setSpanAttributes(span, { "gcs.save_successful": true });
- return;
+ return await saveJobToGCS({
+ mode: "deep_research",
+ id: deepResearch.id,
+ request_id: deepResearch.request_id,
+ team_id: deepResearch.team_id,
+ is_successful: true,
+ num_docs: 1,
+ data: deepResearch.result,
+ zeroDataRetention: false, // ZDR not supported on deep research
+ metadata: {
+ mode: "deep_research",
+ job_id: deepResearch.id,
+ team_id:
+ deepResearch.team_id === "preview" ||
+ deepResearch.team_id?.startsWith("preview_")
+ ? null
+ : deepResearch.team_id,
+ options: JSON.stringify(deepResearch.options),
+ credits_cost: deepResearch.credits_cost,
+ success: true,
+ time_taken: deepResearch.time_taken,
+ },
+ logger: _logger,
});
}
-export async function saveLlmsTxtToGCS(llmsTxt: LoggedLlmsTxt): Promise<void> {
- return await withSpan("firecrawl-gcs-save-llms-txt", async span => {
- setSpanAttributes(span, {
- "gcs.operation": "save_llms_txt",
- "llms_txt.id": llmsTxt.id,
- "llms_txt.team_id": llmsTxt.team_id,
- });
-
- if (!config.GCS_BUCKET_NAME) {
- setSpanAttributes(span, { "gcs.bucket_configured": false });
- return;
- }
-
- const bucket = storage.bucket(config.GCS_BUCKET_NAME);
- const blob = bucket.file(idToFilename(llmsTxt.id));
-
- for (let i = 0; i < 3; i++) {
- try {
- await blob.save(JSON.stringify(llmsTxt.result), {
- contentType: "application/json",
- });
- setSpanAttributes(span, { "gcs.save_successful": true });
- break;
- } catch (error) {
- if (i === 2) {
- throw error;
- } else {
- logger.error(`Error saving llms txt to GCS, retrying`, {
- error,
- llmsTxtId: llmsTxt.id,
- i,
- });
- }
- }
- }
-
- for (let i = 0; i < 3; i++) {
- try {
- await blob.setMetadata({
- metadata: {
- mode: "llms_txt",
- job_id: llmsTxt.id,
- team_id:
- llmsTxt.team_id === "preview" ||
- llmsTxt.team_id?.startsWith("preview_")
- ? null
- : llmsTxt.team_id,
- options: JSON.stringify(llmsTxt.options),
- credits_cost: llmsTxt.credits_cost,
- success: true,
- num_urls: llmsTxt.num_urls,
- cost_tracking: JSON.stringify(llmsTxt.cost_tracking),
- },
- });
- setSpanAttributes(span, { "gcs.save_successful": true });
- break;
- } catch (error) {
- if (i === 2) {
- throw error;
- } else {
- logger.error(`Error saving llms txt metadata to GCS, retrying`, {
- error,
- llmsTxtId: llmsTxt.id,
- i,
- });
- }
- }
- }
-
- setSpanAttributes(span, { "gcs.save_successful": true });
- return;
+export async function saveLlmsTxtToGCS(
+ llmsTxt: LoggedLlmsTxt,
+ _logger: Logger = logger,
+): Promise<void> {
+ return await saveJobToGCS({
+ mode: "llms_txt",
+ id: llmsTxt.id,
+ team_id: llmsTxt.team_id,
+ request_id: llmsTxt.request_id,
+ num_docs: 1,
+ is_successful: true,
+ zeroDataRetention: false, // ZDR not supported on llms txt
+ data: llmsTxt.result,
+ metadata: {
+ mode: "llms_txt",
+ job_id: llmsTxt.id,
+ team_id:
+ llmsTxt.team_id === "preview" || llmsTxt.team_id?.startsWith("preview_")
+ ? null
+ : llmsTxt.team_id,
+ options: JSON.stringify(llmsTxt.options),
+ credits_cost: llmsTxt.credits_cost,
+ success: true,
+ num_urls: llmsTxt.num_urls,
+ cost_tracking: JSON.stringify(llmsTxt.cost_tracking),
+ },
+ logger: _logger,
});
}
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 5241a6d37..bc5363303 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -278,7 +278,7 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
config.GCS_BUCKET_NAME &&
!(scrape.skipNuq && scrape.zeroDataRetention)
) {
- await saveScrapeToGCS(scrape);
+ await saveScrapeToGCS(scrape, logger);
}
if (
@@ -461,7 +461,7 @@ export async function logSearch(search: LoggedSearch, force: boolean = false) {
);
if (search.results && !search.zeroDataRetention) {
- await saveSearchToGCS(search);
+ await saveSearchToGCS(search, logger);
}
}
@@ -514,7 +514,7 @@ export async function logExtract(
if (extract.result) {
if (config.GCS_BUCKET_NAME) {
- await saveExtractToGCS(extract);
+ await saveExtractToGCS(extract, logger);
} else {
// Fallback: save result to Redis with 24h TTL when GCS is not configured
await saveExtractResult(extract.id, extract.result);
@@ -564,7 +564,7 @@ export async function logMap(map: LoggedMap, force: boolean = false) {
);
if (map.results && !map.zeroDataRetention) {
- await saveMapToGCS(map);
+ await saveMapToGCS(map, logger);
}
}
@@ -612,7 +612,7 @@ export async function logLlmsTxt(
);
if (llmsTxt.result) {
- await saveLlmsTxtToGCS(llmsTxt);
+ await saveLlmsTxtToGCS(llmsTxt, logger);
}
}
@@ -661,6 +661,6 @@ export async function logDeepResearch(
);
if (deepResearch.result) {
- await saveDeepResearchToGCS(deepResearch);
+ await saveDeepResearchToGCS(deepResearch, logger);
}
}