Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "Nick:"

Nicolascb764b6dc0c1
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index a28c24bff..edb088d02 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -154,8 +154,7 @@ export async function fireEngineCheckStatus(
const doc = await getDocFromGCS(status.docUrl.split("/").pop() ?? "");
if (doc) {
status = { ...status, ...doc };
- // Keep docUrl so handlers can extract ID from it
- // It's converted to gcsPath in handlers and not carried forward in EngineScrapeResult
+ delete status.docUrl;
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index d99067ee9..b8d575112 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -358,7 +358,6 @@ export async function scrapeURLWithFireEngineChromeCDP(
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
youtubeTranscriptContent: response.youtubeTranscriptContent,
- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
};
});
}
@@ -435,7 +434,6 @@ export async function scrapeURLWithFireEnginePlaywright(
: {}),
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
};
});
}
@@ -501,7 +499,6 @@ export async function scrapeURLWithFireEngineTLSClient(
) ?? [])[1] ?? undefined,
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
- gcsPath: response.docUrl?.split("/").pop(), // Extract just the filename/ID from the path
};
});
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 86145ff3c..743f744ed 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -202,8 +202,7 @@ export async function fireEngineScrape<
const doc = await getDocFromGCS(status.docUrl.split("/").pop() ?? "");
if (doc) {
status = { ...status, ...doc };
- // Keep docUrl so handlers can extract ID from it
- // It's converted to gcsPath in handlers and not carried forward in EngineScrapeResult
+ delete status.docUrl;
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index bad6464e1..309a7c2aa 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -131,8 +131,6 @@ export type EngineScrapeResult = {
postprocessorsUsed?: string[];
proxyUsed: "basic" | "stealth";
-
- gcsPath?: string; // GCS path for raw HTML storage
};
const engineHandlers: {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 7dedc0d40..633b9706f 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -100,7 +100,6 @@ export type Meta = {
| undefined; // undefined: no prefetch yet, null: prefetch came back empty
costTracking: CostTracking;
winnerEngine?: Engine;
- gcsPath?: string; // GCS path for raw HTML storage (from fire-engine)
};
function buildFeatureFlags(
@@ -661,11 +660,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
meta.winnerEngine = result.engine;
let engineResult: EngineScrapeResult = result.result;
-
- // Store GCS path from engine result for downstream use (e.g., search index)
- if (engineResult.gcsPath) {
- meta.gcsPath = engineResult.gcsPath;
- }
for (const postprocessor of postprocessors) {
if (
diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
index 350674de9..4fa44075b 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
@@ -131,7 +131,7 @@ export async function sendDocumentToSearchIndex(
markdown: document.markdown ?? "",
html: document.rawHtml ?? "",
statusCode: document.metadata.statusCode,
- gcsPath: meta.gcsPath ?? undefined, // GCS path from fire-engine
+ gcsPath: undefined, // Can be populated from GCS if needed
screenshotUrl: document.screenshot ?? undefined,
language: document.metadata.language ?? "en",
country: meta.options.location?.country ?? undefined,