Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick:

Nicolasf242cd3034ba
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index edb088d02..a28c24bff 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -154,7 +154,8 @@ export async function fireEngineCheckStatus(
const doc = await getDocFromGCS(status.docUrl.split("/").pop() ?? "");
if (doc) {
status = { ...status, ...doc };
- delete status.docUrl;
+ // Keep docUrl so handlers can extract ID from it
+ // It's converted to gcsPath in handlers and not carried forward in EngineScrapeResult
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index b8d575112..d99067ee9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -358,6 +358,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
youtubeTranscriptContent: response.youtubeTranscriptContent,
+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
};
});
}
@@ -434,6 +435,7 @@ export async function scrapeURLWithFireEnginePlaywright(
: {}),
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
};
});
}
@@ -499,6 +501,7 @@ export async function scrapeURLWithFireEngineTLSClient(
) ?? [])[1] ?? undefined,
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
+ gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
};
});
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 743f744ed..86145ff3c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -202,7 +202,8 @@ export async function fireEngineScrape<
const doc = await getDocFromGCS(status.docUrl.split("/").pop() ?? "");
if (doc) {
status = { ...status, ...doc };
- delete status.docUrl;
+ // Keep docUrl so handlers can extract ID from it
+ // It's converted to gcsPath in handlers and not carried forward in EngineScrapeResult
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 309a7c2aa..bad6464e1 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -131,6 +131,8 @@ export type EngineScrapeResult = {
postprocessorsUsed?: string[];
proxyUsed: "basic" | "stealth";
+
+ gcsPath?: string; // GCS path for raw HTML storage
};
const engineHandlers: {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 633b9706f..7dedc0d40 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -100,6 +100,7 @@ export type Meta = {
| undefined; // undefined: no prefetch yet, null: prefetch came back empty
costTracking: CostTracking;
winnerEngine?: Engine;
+ gcsPath?: string; // GCS path for raw HTML storage (from fire-engine)
};
function buildFeatureFlags(
@@ -660,6 +661,11 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
meta.winnerEngine = result.engine;
let engineResult: EngineScrapeResult = result.result;
+
+ // Store GCS path from engine result for downstream use (e.g., search index)
+ if (engineResult.gcsPath) {
+ meta.gcsPath = engineResult.gcsPath;
+ }
for (const postprocessor of postprocessors) {
if (
diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
index 4fa44075b..350674de9 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
@@ -131,7 +131,7 @@ export async function sendDocumentToSearchIndex(
markdown: document.markdown ?? "",
html: document.rawHtml ?? "",
statusCode: document.metadata.statusCode,
- gcsPath: undefined, // Can be populated from GCS if needed
+ gcsPath: meta.gcsPath ?? undefined, // GCS path from fire-engine
screenshotUrl: document.screenshot ?? undefined,
language: document.metadata.language ?? "en",
country: meta.options.location?.country ?? undefined,