Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/scrapeURL): better contentType tracking

mogery070ede0491ff
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
index 51582e813..c7fc89e8b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
@@ -73,6 +73,21 @@ function getDocumentTypeFromContentType(
return null;
}
+function getContentTypeFromDocumentType(documentType: DocumentType): string {
+ switch (documentType) {
+ case DocumentType.Docx:
+ return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+ case DocumentType.Doc:
+ return "application/msword";
+ case DocumentType.Odt:
+ return "application/vnd.oasis.opendocument.text";
+ case DocumentType.Rtf:
+ return "application/rtf";
+ case DocumentType.Xlsx:
+ return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+ }
+}
+
function isValidDocumentContentType(contentType: string | null): boolean {
if (!contentType) return false;
@@ -149,7 +164,9 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
url: response.url,
statusCode: response.status,
html,
- contentType: response.headers.get("content-type") ?? undefined,
+ contentType:
+ response.headers.get("content-type") ??
+ getContentTypeFromDocumentType(documentType),
proxyUsed,
};
} finally {
diff --git a/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts b/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts
index 1f6a17e7b..f479ad7df 100644
--- a/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts
@@ -6,17 +6,13 @@ import { EngineError } from "../../error";
import { getRedisConnection } from "../../../../services/queue-service";
import { redlock } from "../../../../services/redlock";
-const WIKIMEDIA_AUTH_URL =
- "https://auth.enterprise.wikimedia.com/v1/login";
-const WIKIMEDIA_API_BASE =
- "https://api.enterprise.wikimedia.com/v2";
+const WIKIMEDIA_AUTH_URL = "https://auth.enterprise.wikimedia.com/v1/login";
+const WIKIMEDIA_API_BASE = "https://api.enterprise.wikimedia.com/v2";
const REDIS_TOKEN_KEY = "wikipedia_enterprise:access_token";
const REDIS_AUTH_LOCK_KEY = "lock:wikipedia_enterprise:auth";
-async function getAccessToken(
- logger: Meta["logger"],
-): Promise<string> {
+async function getAccessToken(logger: Meta["logger"]): Promise<string> {
const redis = getRedisConnection();
try {
@@ -25,74 +21,75 @@ async function getAccessToken(
return cached;
}
} catch (error) {
- logger.warn("Failed to read Wikipedia token from Redis, will re-authenticate", { error });
+ logger.warn(
+ "Failed to read Wikipedia token from Redis, will re-authenticate",
+ { error },
+ );
}
// Acquire a distributed lock so only one instance authenticates at a time.
// Others wait and then read the token from Redis.
- return await redlock.using(
- [REDIS_AUTH_LOCK_KEY],
- 10000,
- async (signal) => {
- // Double-check: another instance may have authenticated while we waited for the lock
- try {
- const cached = await redis.get(REDIS_TOKEN_KEY);
- if (cached) {
- return cached;
- }
- } catch {}
-
- const username = config.WIKIPEDIA_ENTERPRISE_USERNAME;
- const password = config.WIKIPEDIA_ENTERPRISE_PASSWORD;
-
- if (!username || !password) {
- throw new EngineError(
- "Wikipedia Enterprise API credentials are not configured (WIKIPEDIA_ENTERPRISE_USERNAME, WIKIPEDIA_ENTERPRISE_PASSWORD)",
- );
+ return await redlock.using([REDIS_AUTH_LOCK_KEY], 10000, async signal => {
+ // Double-check: another instance may have authenticated while we waited for the lock
+ try {
+ const cached = await redis.get(REDIS_TOKEN_KEY);
+ if (cached) {
+ return cached;
}
+ } catch {}
- logger.info("Authenticating with Wikipedia Enterprise API");
+ const username = config.WIKIPEDIA_ENTERPRISE_USERNAME;
+ const password = config.WIKIPEDIA_ENTERPRISE_PASSWORD;
- const response = await undici.fetch(WIKIMEDIA_AUTH_URL, {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ username, password }),
- });
+ if (!username || !password) {
+ throw new EngineError(
+ "Wikipedia Enterprise API credentials are not configured (WIKIPEDIA_ENTERPRISE_USERNAME, WIKIPEDIA_ENTERPRISE_PASSWORD)",
+ );
+ }
- if (signal.aborted) {
- throw signal.error;
- }
+ logger.info("Authenticating with Wikipedia Enterprise API");
- if (!response.ok) {
- const body = await response.text();
- throw new EngineError(
- `Wikipedia Enterprise authentication failed (${response.status}): ${body}`,
- );
- }
+ const response = await undici.fetch(WIKIMEDIA_AUTH_URL, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ username, password }),
+ });
- const data = (await response.json()) as {
- access_token: string;
- expires_in: number;
- };
+ if (signal.aborted) {
+ throw signal.error;
+ }
- const ttlSeconds = Math.max(data.expires_in - 300, 60);
+ if (!response.ok) {
+ const body = await response.text();
+ throw new EngineError(
+ `Wikipedia Enterprise authentication failed (${response.status}): ${body}`,
+ );
+ }
- try {
- await redis.set(REDIS_TOKEN_KEY, data.access_token, "EX", ttlSeconds);
- } catch (error) {
- logger.warn("Failed to cache Wikipedia token in Redis", { error });
- }
+ const data = (await response.json()) as {
+ access_token: string;
+ expires_in: number;
+ };
- return data.access_token;
- },
- );
+ const ttlSeconds = Math.max(data.expires_in - 300, 60);
+
+ try {
+ await redis.set(REDIS_TOKEN_KEY, data.access_token, "EX", ttlSeconds);
+ } catch (error) {
+ logger.warn("Failed to cache Wikipedia token in Redis", { error });
+ }
+
+ return data.access_token;
+ });
}
function clearCachedToken(logger: Meta["logger"]): void {
try {
- getRedisConnection().del(REDIS_TOKEN_KEY).catch((error) => {
- logger.warn("Failed to clear Wikipedia token from Redis", { error });
- });
+ getRedisConnection()
+ .del(REDIS_TOKEN_KEY)
+ .catch(error => {
+ logger.warn("Failed to clear Wikipedia token from Redis", { error });
+ });
} catch {
logger.warn("Failed to clear Wikipedia token from Redis");
}
@@ -153,7 +150,8 @@ async function resolveRedirect(
domain: string,
logger: Meta["logger"],
): Promise<string> {
- const apiUrl = `https://${lang}.${domain}/w/api.php?` +
+ const apiUrl =
+ `https://${lang}.${domain}/w/api.php?` +
new URLSearchParams({
action: "query",
titles: articleName,
@@ -311,18 +309,14 @@ export async function scrapeURLWithWikipedia(
const html = article.article_body?.html;
if (!html) {
- throw new EngineError(
- `Wikipedia article has no HTML body: ${articleName}`,
- );
+ throw new EngineError(`Wikipedia article has no HTML body: ${articleName}`);
}
// Build a full HTML document wrapping the article content with metadata
const categories = (article.categories ?? [])
.map(c => `<li><a href="${c.url}">${c.name}</a></li>`)
.join("");
- const toc = (article.has_parts ?? [])
- .map(p => `<li>${p.name}</li>`)
- .join("");
+ const toc = (article.has_parts ?? []).map(p => `<li>${p.name}</li>`).join("");
const fullHtml = `<!DOCTYPE html>
<html lang="${lang}">
@@ -357,6 +351,7 @@ export async function scrapeURLWithWikipedia(
url: article.url ?? urlToScrape,
html: fullHtml,
statusCode: 200,
+ contentType: "text/html; charset=utf-8",
proxyUsed: "basic",
};
}