Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/scrapeURL): better contentType tracking
mogery070ede0491ff
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.tsindex 51582e813..c7fc89e8b 100644--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts@@ -73,6 +73,21 @@ function getDocumentTypeFromContentType(return null;}+function getContentTypeFromDocumentType(documentType: DocumentType): string {+ switch (documentType) {+ case DocumentType.Docx:+ return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";+ case DocumentType.Doc:+ return "application/msword";+ case DocumentType.Odt:+ return "application/vnd.oasis.opendocument.text";+ case DocumentType.Rtf:+ return "application/rtf";+ case DocumentType.Xlsx:+ return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";+ }+}+function isValidDocumentContentType(contentType: string | null): boolean {if (!contentType) return false;@@ -149,7 +164,9 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {url: response.url,statusCode: response.status,html,- contentType: response.headers.get("content-type") ?? undefined,+ contentType:+ response.headers.get("content-type") ??+ getContentTypeFromDocumentType(documentType),proxyUsed,};} finally {diff --git a/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts b/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.tsindex 1f6a17e7b..f479ad7df 100644--- a/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts@@ -6,17 +6,13 @@ import { EngineError } from "../../error";import { getRedisConnection } from "../../../../services/queue-service";import { redlock } from "../../../../services/redlock";-const WIKIMEDIA_AUTH_URL =- "https://auth.enterprise.wikimedia.com/v1/login";-const WIKIMEDIA_API_BASE =- "https://api.enterprise.wikimedia.com/v2";+const WIKIMEDIA_AUTH_URL = "https://auth.enterprise.wikimedia.com/v1/login";+const WIKIMEDIA_API_BASE = "https://api.enterprise.wikimedia.com/v2";const REDIS_TOKEN_KEY = "wikipedia_enterprise:access_token";const REDIS_AUTH_LOCK_KEY = "lock:wikipedia_enterprise:auth";-async function getAccessToken(- logger: Meta["logger"],-): Promise<string> {+async function getAccessToken(logger: Meta["logger"]): Promise<string> {const redis = getRedisConnection();try {@@ -25,74 +21,75 @@ async function getAccessToken(return cached;}} catch (error) {- logger.warn("Failed to read Wikipedia token from Redis, will re-authenticate", { error });+ logger.warn(+ "Failed to read Wikipedia token from Redis, will re-authenticate",+ { error },+ );}// Acquire a distributed lock so only one instance authenticates at a time.// Others wait and then read the token from Redis.- return await redlock.using(- [REDIS_AUTH_LOCK_KEY],- 10000,- async (signal) => {- // Double-check: another instance may have authenticated while we waited for the lock- try {- const cached = await redis.get(REDIS_TOKEN_KEY);- if (cached) {- return cached;- }- } catch {}-- const username = config.WIKIPEDIA_ENTERPRISE_USERNAME;- const password = config.WIKIPEDIA_ENTERPRISE_PASSWORD;-- if (!username || !password) {- throw new EngineError(- "Wikipedia Enterprise API credentials are not configured (WIKIPEDIA_ENTERPRISE_USERNAME, WIKIPEDIA_ENTERPRISE_PASSWORD)",- );+ return await redlock.using([REDIS_AUTH_LOCK_KEY], 10000, async signal => {+ // Double-check: another instance may have authenticated while we waited for the lock+ try {+ const cached = await redis.get(REDIS_TOKEN_KEY);+ if (cached) {+ return cached;}+ } catch {}- logger.info("Authenticating with Wikipedia Enterprise API");+ const username = config.WIKIPEDIA_ENTERPRISE_USERNAME;+ const password = config.WIKIPEDIA_ENTERPRISE_PASSWORD;- const response = await undici.fetch(WIKIMEDIA_AUTH_URL, {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify({ username, password }),- });+ if (!username || !password) {+ throw new EngineError(+ "Wikipedia Enterprise API credentials are not configured (WIKIPEDIA_ENTERPRISE_USERNAME, WIKIPEDIA_ENTERPRISE_PASSWORD)",+ );+ }- if (signal.aborted) {- throw signal.error;- }+ logger.info("Authenticating with Wikipedia Enterprise API");- if (!response.ok) {- const body = await response.text();- throw new EngineError(- `Wikipedia Enterprise authentication failed (${response.status}): ${body}`,- );- }+ const response = await undici.fetch(WIKIMEDIA_AUTH_URL, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({ username, password }),+ });- const data = (await response.json()) as {- access_token: string;- expires_in: number;- };+ if (signal.aborted) {+ throw signal.error;+ }- const ttlSeconds = Math.max(data.expires_in - 300, 60);+ if (!response.ok) {+ const body = await response.text();+ throw new EngineError(+ `Wikipedia Enterprise authentication failed (${response.status}): ${body}`,+ );+ }- try {- await redis.set(REDIS_TOKEN_KEY, data.access_token, "EX", ttlSeconds);- } catch (error) {- logger.warn("Failed to cache Wikipedia token in Redis", { error });- }+ const data = (await response.json()) as {+ access_token: string;+ expires_in: number;+ };- return data.access_token;- },- );+ const ttlSeconds = Math.max(data.expires_in - 300, 60);++ try {+ await redis.set(REDIS_TOKEN_KEY, data.access_token, "EX", ttlSeconds);+ } catch (error) {+ logger.warn("Failed to cache Wikipedia token in Redis", { error });+ }++ return data.access_token;+ });}function clearCachedToken(logger: Meta["logger"]): void {try {- getRedisConnection().del(REDIS_TOKEN_KEY).catch((error) => {- logger.warn("Failed to clear Wikipedia token from Redis", { error });- });+ getRedisConnection()+ .del(REDIS_TOKEN_KEY)+ .catch(error => {+ logger.warn("Failed to clear Wikipedia token from Redis", { error });+ });} catch {logger.warn("Failed to clear Wikipedia token from Redis");}@@ -153,7 +150,8 @@ async function resolveRedirect(domain: string,logger: Meta["logger"],): Promise<string> {- const apiUrl = `https://${lang}.${domain}/w/api.php?` ++ const apiUrl =+ `https://${lang}.${domain}/w/api.php?` +new URLSearchParams({action: "query",titles: articleName,@@ -311,18 +309,14 @@ export async function scrapeURLWithWikipedia(const html = article.article_body?.html;if (!html) {- throw new EngineError(- `Wikipedia article has no HTML body: ${articleName}`,- );+ throw new EngineError(`Wikipedia article has no HTML body: ${articleName}`);}// Build a full HTML document wrapping the article content with metadataconst categories = (article.categories ?? []).map(c => `<li><a href="${c.url}">${c.name}</a></li>`).join("");- const toc = (article.has_parts ?? [])- .map(p => `<li>${p.name}</li>`)- .join("");+ const toc = (article.has_parts ?? []).map(p => `<li>${p.name}</li>`).join("");const fullHtml = `<!DOCTYPE html><html lang="${lang}">@@ -357,6 +351,7 @@ export async function scrapeURLWithWikipedia(url: article.url ?? urlToScrape,html: fullHtml,statusCode: 200,+ contentType: "text/html; charset=utf-8",proxyUsed: "basic",};}