Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(feat/wiki) Wikimedia (#3121)
Nicolas80e8d708fecd
diff --git a/apps/api/src/__tests__/snips/v2/wikipedia.test.ts b/apps/api/src/__tests__/snips/v2/wikipedia.test.tsnew file mode 100644index 000000000..dc9f24f53--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/wikipedia.test.ts@@ -0,0 +1,329 @@+import { describeIf } from "../lib";+import {+ scrape,+ scrapeRaw,+ scrapeTimeout,+ idmux,+ Identity,+} from "./lib";++const HAS_WIKIPEDIA = !!(+ process.env.WIKIPEDIA_ENTERPRISE_USERNAME &&+ process.env.WIKIPEDIA_ENTERPRISE_PASSWORD+);++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "wikipedia",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000);++describeIf(HAS_WIKIPEDIA && !process.env.TEST_SUITE_SELF_HOSTED)(+ "Wikipedia Enterprise API integration",+ () => {+ describe("basic scraping", () => {+ it.concurrent(+ "scrapes a Wikipedia article and returns markdown by default",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/Web_scraping",+ },+ identity,+ );++ expect(response.markdown).toBeTruthy();+ expect(response.markdown!.length).toBeGreaterThan(100);+ expect(response.metadata.statusCode).toBe(200);+ expect(response.metadata.sourceURL).toBe(+ "https://en.wikipedia.org/wiki/Web_scraping",+ );+ },+ scrapeTimeout,+ );++ it.concurrent(+ "scrapes a non-English Wikipedia article (Spanish)",+ async () => {+ const response = await scrape(+ {+ url: "https://es.wikipedia.org/wiki/Web_scraping",+ },+ identity,+ );++ expect(response.markdown).toBeTruthy();+ expect(response.markdown!.length).toBeGreaterThan(50);+ expect(response.metadata.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "scrapes a non-English Wikipedia article (French)",+ async () => {+ const response = await scrape(+ {+ url: "https://fr.wikipedia.org/wiki/France",+ },+ identity,+ );++ expect(response.markdown).toBeTruthy();+ expect(response.markdown!.length).toBeGreaterThan(100);+ expect(response.metadata.statusCode).toBe(200);+ },+ scrapeTimeout,+ );+ });++ describe("redirect handling", () => {+ it.concurrent(+ "handles Wikipedia redirect pages (e.g. Brasil -> Brazil)",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/Brasil",+ },+ identity,+ );++ expect(response.markdown).toBeTruthy();+ expect(response.markdown!.length).toBeGreaterThan(100);+ expect(response.metadata.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "handles abbreviation redirects (e.g. UK -> United Kingdom)",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/UK",+ },+ identity,+ );++ expect(response.markdown).toBeTruthy();+ expect(response.markdown!.length).toBeGreaterThan(100);+ expect(response.metadata.statusCode).toBe(200);+ },+ scrapeTimeout,+ );+ });++ describe("special characters", () => {+ it.concurrent(+ "handles URL-encoded special characters (C++)",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/C%2B%2B",+ },+ identity,+ );++ expect(response.markdown).toBeTruthy();+ expect(response.markdown!.length).toBeGreaterThan(100);+ expect(response.metadata.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "handles articles with parentheses (disambiguation)",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/Mercury_(planet)",+ },+ identity,+ );++ expect(response.markdown).toBeTruthy();+ expect(response.markdown!.length).toBeGreaterThan(100);+ expect(response.metadata.statusCode).toBe(200);+ },+ scrapeTimeout,+ );+ });++ describe("output formats", () => {+ it.concurrent(+ "returns markdown when format is markdown",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/TypeScript",+ formats: ["markdown"],+ },+ identity,+ );++ expect(response.markdown).toBeTruthy();+ expect(response.markdown!.length).toBeGreaterThan(100);+ expect(response.markdown).toContain("TypeScript");+ expect(response.html).toBeUndefined();+ expect(response.rawHtml).toBeUndefined();+ },+ scrapeTimeout,+ );++ it.concurrent(+ "returns html when format is html",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/TypeScript",+ formats: ["html"],+ },+ identity,+ );++ expect(response.html).toBeTruthy();+ expect(response.html!.length).toBeGreaterThan(100);+ expect(response.html).toContain("TypeScript");+ expect(response.markdown).toBeUndefined();+ },+ scrapeTimeout,+ );++ it.concurrent(+ "returns rawHtml when format is rawHtml",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/TypeScript",+ formats: ["rawHtml"],+ },+ identity,+ );++ expect(response.rawHtml).toBeTruthy();+ expect(response.rawHtml!.length).toBeGreaterThan(100);+ expect(response.rawHtml).toContain("TypeScript");+ expect(response.markdown).toBeUndefined();+ },+ scrapeTimeout,+ );++ it.concurrent(+ "returns links when format is links",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/TypeScript",+ formats: ["links"],+ },+ identity,+ );++ expect(response.links).toBeTruthy();+ expect(Array.isArray(response.links)).toBe(true);+ expect(response.links!.length).toBeGreaterThan(0);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "returns multiple formats together",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/TypeScript",+ formats: ["markdown", "html", "links"],+ },+ identity,+ );++ expect(response.markdown).toBeTruthy();+ expect(response.html).toBeTruthy();+ expect(response.links).toBeTruthy();+ expect(response.markdown!.length).toBeGreaterThan(100);+ expect(response.html!.length).toBeGreaterThan(100);+ expect(response.links!.length).toBeGreaterThan(0);+ },+ scrapeTimeout,+ );+ });++ describe("metadata", () => {+ it.concurrent(+ "returns correct metadata for a Wikipedia article",+ async () => {+ const response = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/TypeScript",+ formats: ["markdown"],+ },+ identity,+ );++ expect(response.metadata).toBeDefined();+ expect(response.metadata.statusCode).toBe(200);+ expect(response.metadata.sourceURL).toBe(+ "https://en.wikipedia.org/wiki/TypeScript",+ );+ expect(response.metadata.title).toBeTruthy();+ expect(response.metadata.description).toBeTruthy();+ },+ scrapeTimeout,+ );+ });++ describe("error handling", () => {+ it.concurrent(+ "fails gracefully for non-existent Wikipedia article",+ async () => {+ const response = await scrapeRaw(+ {+ url: "https://en.wikipedia.org/wiki/ThisArticleDefinitelyDoesNotExist_XYZZY_123456789",+ },+ identity,+ );++ expect(response.statusCode).not.toBe(200);+ },+ scrapeTimeout,+ );+ });++ describe("scrape options", () => {+ it.concurrent(+ "respects onlyMainContent option",+ async () => {+ const withMainContent = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/TypeScript",+ formats: ["markdown"],+ onlyMainContent: true,+ },+ identity,+ );++ const withoutMainContent = await scrape(+ {+ url: "https://en.wikipedia.org/wiki/TypeScript",+ formats: ["markdown"],+ onlyMainContent: false,+ },+ identity,+ );++ expect(withMainContent.markdown).toBeTruthy();+ expect(withoutMainContent.markdown).toBeTruthy();+ // Full content should be equal to or longer than main-only content+ expect(withoutMainContent.markdown!.length).toBeGreaterThanOrEqual(+ withMainContent.markdown!.length,+ );+ },+ scrapeTimeout * 2,+ );+ });+ },+);diff --git a/apps/api/src/__tests__/snips/wikipedia-url-parser.test.ts b/apps/api/src/__tests__/snips/wikipedia-url-parser.test.tsnew file mode 100644index 000000000..7b30ad7d2--- /dev/null+++ b/apps/api/src/__tests__/snips/wikipedia-url-parser.test.ts@@ -0,0 +1,349 @@+import {+ parseWikimediaUrl,+ isWikimediaUrl,+} from "../../scraper/scrapeURL/engines/wikipedia";++describe("Wikipedia URL parser", () => {+ describe("parseWikimediaUrl", () => {+ describe("Wikipedia articles", () => {+ it("parses English Wikipedia article URL", () => {+ const result = parseWikimediaUrl(+ "https://en.wikipedia.org/wiki/Web_scraping",+ );+ expect(result).toEqual({+ articleName: "Web_scraping",+ lang: "en",+ projectIdentifier: "enwiki",+ });+ });++ it("parses German Wikipedia article URL", () => {+ const result = parseWikimediaUrl(+ "https://de.wikipedia.org/wiki/Web_Scraping",+ );+ expect(result).toEqual({+ articleName: "Web_Scraping",+ lang: "de",+ projectIdentifier: "dewiki",+ });+ });++ it("parses Spanish Wikipedia article URL", () => {+ const result = parseWikimediaUrl(+ "https://es.wikipedia.org/wiki/Scraping",+ );+ expect(result).toEqual({+ articleName: "Scraping",+ lang: "es",+ projectIdentifier: "eswiki",+ });+ });++ it("parses Japanese Wikipedia (3-letter lang code)", () => {+ const result = parseWikimediaUrl(+ "https://ja.wikipedia.org/wiki/ζ₯ζ¬",+ );+ expect(result).toEqual({+ articleName: "ζ₯ζ¬",+ lang: "ja",+ projectIdentifier: "jawiki",+ });+ });++ it("parses Portuguese Wikipedia", () => {+ const result = parseWikimediaUrl(+ "https://pt.wikipedia.org/wiki/Brasil",+ );+ expect(result).toEqual({+ articleName: "Brasil",+ lang: "pt",+ projectIdentifier: "ptwiki",+ });+ });+ });++ describe("other Wikimedia projects", () => {+ it("parses Wiktionary URL", () => {+ const result = parseWikimediaUrl(+ "https://fr.wiktionary.org/wiki/bonjour",+ );+ expect(result).toEqual({+ articleName: "bonjour",+ lang: "fr",+ projectIdentifier: "frwiktionary",+ });+ });++ it("parses Wikisource URL", () => {+ const result = parseWikimediaUrl(+ "https://en.wikisource.org/wiki/The_Art_of_War",+ );+ expect(result).toEqual({+ articleName: "The_Art_of_War",+ lang: "en",+ projectIdentifier: "enwikisource",+ });+ });++ it("parses Wikibooks URL", () => {+ const result = parseWikimediaUrl(+ "https://en.wikibooks.org/wiki/Cookbook:Table_of_Contents",+ );+ expect(result).toEqual({+ articleName: "Cookbook:Table_of_Contents",+ lang: "en",+ projectIdentifier: "enwikibooks",+ });+ });++ it("parses Wikiquote URL", () => {+ const result = parseWikimediaUrl(+ "https://en.wikiquote.org/wiki/Albert_Einstein",+ );+ expect(result).toEqual({+ articleName: "Albert_Einstein",+ lang: "en",+ projectIdentifier: "enwikiquote",+ });+ });++ it("parses Wikiversity URL", () => {+ const result = parseWikimediaUrl(+ "https://en.wikiversity.org/wiki/Introduction_to_Computers",+ );+ expect(result).toEqual({+ articleName: "Introduction_to_Computers",+ lang: "en",+ projectIdentifier: "enwikiversity",+ });+ });++ it("parses Wikivoyage URL", () => {+ const result = parseWikimediaUrl(+ "https://en.wikivoyage.org/wiki/Paris",+ );+ expect(result).toEqual({+ articleName: "Paris",+ lang: "en",+ projectIdentifier: "enwikivoyage",+ });+ });+ });++ describe("special characters and encoding", () => {+ it("handles URL-encoded article names", () => {+ const result = parseWikimediaUrl(+ "https://en.wikipedia.org/wiki/C%2B%2B",+ );+ expect(result).toEqual({+ articleName: "C++",+ lang: "en",+ projectIdentifier: "enwiki",+ });+ });++ it("handles articles with underscores (spaces)", () => {+ const result = parseWikimediaUrl(+ "https://en.wikipedia.org/wiki/United_States",+ );+ expect(result).toEqual({+ articleName: "United_States",+ lang: "en",+ projectIdentifier: "enwiki",+ });+ });++ it("handles articles with parentheses (disambiguation)", () => {+ const result = parseWikimediaUrl(+ "https://en.wikipedia.org/wiki/Mercury_(planet)",+ );+ expect(result).toEqual({+ articleName: "Mercury_(planet)",+ lang: "en",+ projectIdentifier: "enwiki",+ });+ });++ it("handles URL-encoded parentheses", () => {+ const result = parseWikimediaUrl(+ "https://en.wikipedia.org/wiki/Mercury_%28planet%29",+ );+ expect(result).toEqual({+ articleName: "Mercury_(planet)",+ lang: "en",+ projectIdentifier: "enwiki",+ });+ });++ it("handles articles with apostrophes", () => {+ const result = parseWikimediaUrl(+ "https://en.wikipedia.org/wiki/Murphy's_law",+ );+ expect(result).toEqual({+ articleName: "Murphy's_law",+ lang: "en",+ projectIdentifier: "enwiki",+ });+ });++ it("handles articles with colons (namespaces)", () => {+ const result = parseWikimediaUrl(+ "https://en.wikipedia.org/wiki/Wikipedia:About",+ );+ expect(result).toEqual({+ articleName: "Wikipedia:About",+ lang: "en",+ projectIdentifier: "enwiki",+ });+ });++ it("handles articles with URL-encoded spaces (%20)", () => {+ const result = parseWikimediaUrl(+ "https://en.wikipedia.org/wiki/New%20York%20City",+ );+ expect(result).toEqual({+ articleName: "New_York_City",+ lang: "en",+ projectIdentifier: "enwiki",+ });+ });++ it("handles articles with hash/fragment (strips it)", () => {+ const result = parseWikimediaUrl(+ "https://en.wikipedia.org/wiki/Brazil#History",+ );+ expect(result).toEqual({+ articleName: "Brazil",+ lang: "en",+ projectIdentifier: "enwiki",+ });+ });++ it("handles articles with query parameters", () => {+ const result = parseWikimediaUrl(+ "https://en.wikipedia.org/wiki/Brazil?action=edit",+ );+ expect(result).toEqual({+ articleName: "Brazil",+ lang: "en",+ projectIdentifier: "enwiki",+ });+ });+ });++ describe("rejected URLs", () => {+ it("returns null for non-Wikipedia URLs", () => {+ expect(parseWikimediaUrl("https://google.com")).toBeNull();+ expect(+ parseWikimediaUrl("https://example.com/wiki/test"),+ ).toBeNull();+ });++ it("returns null for Wikipedia URLs without /wiki/ path", () => {+ expect(+ parseWikimediaUrl("https://en.wikipedia.org/w/index.php"),+ ).toBeNull();+ expect(parseWikimediaUrl("https://en.wikipedia.org/")).toBeNull();+ });++ it("returns null for main Wikipedia domain without language prefix", () => {+ expect(+ parseWikimediaUrl("https://wikipedia.org/wiki/Test"),+ ).toBeNull();+ });++ it("returns null for invalid URLs", () => {+ expect(parseWikimediaUrl("not-a-url")).toBeNull();+ expect(parseWikimediaUrl("")).toBeNull();+ });++ it("returns null for Wikimedia Commons (not a content project)", () => {+ expect(+ parseWikimediaUrl("https://commons.wikimedia.org/wiki/File:Test.jpg"),+ ).toBeNull();+ });++ it("returns null for Wikidata", () => {+ expect(+ parseWikimediaUrl("https://www.wikidata.org/wiki/Q42"),+ ).toBeNull();+ });++ it("returns null for MediaWiki API endpoints", () => {+ expect(+ parseWikimediaUrl(+ "https://en.wikipedia.org/w/api.php?action=query&titles=Brazil",+ ),+ ).toBeNull();+ });++ it("returns null for Wikipedia Special pages", () => {+ expect(+ parseWikimediaUrl(+ "https://en.wikipedia.org/wiki/Special:Random",+ ),+ ).not.toBeNull(); // Special:Random is technically a valid /wiki/ path+ });++ it("returns null for bare /wiki/ path with no article", () => {+ expect(+ parseWikimediaUrl("https://en.wikipedia.org/wiki/"),+ ).toBeNull();+ });++ it("returns null for HTTP protocol", () => {+ const result = parseWikimediaUrl(+ "http://en.wikipedia.org/wiki/Test",+ );+ // HTTP should still parse -- the protocol doesn't matter for article resolution+ expect(result).toEqual({+ articleName: "Test",+ lang: "en",+ projectIdentifier: "enwiki",+ });+ });+ });+ });++ describe("isWikimediaUrl", () => {+ it("returns true for Wikipedia URLs in various languages", () => {+ expect(isWikimediaUrl("https://en.wikipedia.org/wiki/Test")).toBe(true);+ expect(isWikimediaUrl("https://de.wikipedia.org/wiki/Test")).toBe(true);+ expect(isWikimediaUrl("https://ja.wikipedia.org/wiki/γγΉγ")).toBe(true);+ expect(isWikimediaUrl("https://pt.wikipedia.org/wiki/Brasil")).toBe(true);+ expect(isWikimediaUrl("https://zh.wikipedia.org/wiki/ζ΅θ―")).toBe(true);+ });++ it("returns true for all supported Wikimedia projects", () => {+ expect(isWikimediaUrl("https://en.wiktionary.org/wiki/hello")).toBe(true);+ expect(isWikimediaUrl("https://en.wikisource.org/wiki/Test")).toBe(true);+ expect(isWikimediaUrl("https://en.wikibooks.org/wiki/Test")).toBe(true);+ expect(isWikimediaUrl("https://en.wikiquote.org/wiki/Test")).toBe(true);+ expect(isWikimediaUrl("https://en.wikiversity.org/wiki/Test")).toBe(true);+ expect(isWikimediaUrl("https://en.wikivoyage.org/wiki/Test")).toBe(true);+ });++ it("returns false for non-Wikimedia URLs", () => {+ expect(isWikimediaUrl("https://google.com")).toBe(false);+ expect(isWikimediaUrl("https://example.com/wiki/test")).toBe(false);+ expect(isWikimediaUrl("https://en.fakepedia.org/wiki/test")).toBe(false);+ });++ it("returns false for Wikipedia non-article pages", () => {+ expect(isWikimediaUrl("https://en.wikipedia.org/w/index.php")).toBe(+ false,+ );+ expect(isWikimediaUrl("https://en.wikipedia.org/")).toBe(false);+ });++ it("returns false for unsupported Wikimedia projects", () => {+ expect(+ isWikimediaUrl("https://commons.wikimedia.org/wiki/File:Test.jpg"),+ ).toBe(false);+ expect(isWikimediaUrl("https://www.wikidata.org/wiki/Q42")).toBe(false);+ expect(+ isWikimediaUrl("https://meta.wikimedia.org/wiki/Main_Page"),+ ).toBe(false);+ });+ });+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 4ee2f1214..7f0665263 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -230,6 +230,10 @@ const configSchema = z.object({EXTRACT_V3_BETA_URL: z.string().optional(),AGENT_INTEROP_SECRET: z.string().optional(),+ // Wikipedia Enterprise API+ WIKIPEDIA_ENTERPRISE_USERNAME: z.string().optional(),+ WIKIPEDIA_ENTERPRISE_PASSWORD: z.string().optional(),+// Browser ServiceBROWSER_SERVICE_URL: z.string().optional(),BROWSER_SERVICE_API_KEY: z.string().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 2378e4a52..97765f6b3 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -15,6 +15,11 @@ import {scrapeURLWithPlaywright,} from "./playwright";import { indexMaxReasonableTime, scrapeURLWithIndex } from "./index/index";+import {+ scrapeURLWithWikipedia,+ wikipediaMaxReasonableTime,+ isWikimediaUrl,+} from "./wikipedia";import { queryEngpickerVerdict, useIndex } from "../../../services";import { hasFormatOfType } from "../../../lib/format-utils";import { getPDFMaxPages } from "../../../controllers/v2/types";@@ -36,7 +41,8 @@ export type Engine =| "pdf"| "document"| "index"- | "index;documents";+ | "index;documents"+ | "wikipedia";const useFireEngine =config.FIRE_ENGINE_BETA_URL !== "" &&@@ -44,8 +50,14 @@ const useFireEngine =const usePlaywright =config.PLAYWRIGHT_MICROSERVICE_URL !== "" &&config.PLAYWRIGHT_MICROSERVICE_URL !== undefined;+const useWikipedia =+ config.WIKIPEDIA_ENTERPRISE_USERNAME !== undefined &&+ config.WIKIPEDIA_ENTERPRISE_USERNAME !== "" &&+ config.WIKIPEDIA_ENTERPRISE_PASSWORD !== undefined &&+ config.WIKIPEDIA_ENTERPRISE_PASSWORD !== "";const engines: Engine[] = [+ ...(useWikipedia ? ["wikipedia" as const] : []),...(useIndex ? ["index" as const, "index;documents" as const] : []),...(useFireEngine? [@@ -158,6 +170,7 @@ const engineHandlers: {fetch: scrapeURLWithFetch,pdf: scrapePDF,document: scrapeDocument,+ wikipedia: scrapeURLWithWikipedia,};const engineMRTs: {@@ -185,6 +198,7 @@ const engineMRTs: {fetch: fetchMaxReasonableTime,pdf: pdfMaxReasonableTime,document: documentMaxReasonableTime,+ wikipedia: wikipediaMaxReasonableTime,};const engineOptions: {@@ -463,6 +477,25 @@ const engineOptions: {},quality: -20,},+ wikipedia: {+ features: {+ actions: false,+ waitFor: false,+ screenshot: false,+ "screenshot@fullScreen": false,+ pdf: false,+ document: false,+ atsv: false,+ location: false,+ mobile: false,+ skipTlsVerification: true,+ useFastMode: true,+ stealthProxy: false,+ branding: false,+ disableAdblock: true,+ },+ quality: 500, // below index (1000) so cache is tried first, above fire-engine (50)+ },};export function shouldUseIndex(meta: Meta) {@@ -529,6 +562,13 @@ export async function buildFallbackList(meta: Meta): Promise<}}+ if (!isWikimediaUrl(meta.url)) {+ const wikiIndex = _engines.indexOf("wikipedia");+ if (wikiIndex !== -1) {+ _engines.splice(wikiIndex, 1);+ }+ }+const prioritySum = [...meta.featureFlags].reduce((a, x) => a + featureFlagOptions[x].priority,0,diff --git a/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts b/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.tsnew file mode 100644index 000000000..77ff5a117--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts@@ -0,0 +1,372 @@+import * as undici from "undici";+import { EngineScrapeResult } from "..";+import { Meta } from "../..";+import { config } from "../../../../config";+import { EngineError } from "../../error";+import { getRedisConnection } from "../../../../services/queue-service";+import { redlock } from "../../../../services/redlock";++const WIKIMEDIA_AUTH_URL =+ "https://auth.enterprise.wikimedia.com/v1/login";+const WIKIMEDIA_API_BASE =+ "https://api.enterprise.wikimedia.com/v2";++const REDIS_TOKEN_KEY = "wikipedia_enterprise:access_token";+const REDIS_AUTH_LOCK_KEY = "lock:wikipedia_enterprise:auth";++async function getAccessToken(+ logger: Meta["logger"],+): Promise<string> {+ const redis = getRedisConnection();++ try {+ const cached = await redis.get(REDIS_TOKEN_KEY);+ if (cached) {+ return cached;+ }+ } catch (error) {+ logger.warn("Failed to read Wikipedia token from Redis, will re-authenticate", { error });+ }++ // Acquire a distributed lock so only one instance authenticates at a time.+ // Others wait and then read the token from Redis.+ return await redlock.using(+ [REDIS_AUTH_LOCK_KEY],+ 10000,+ async (signal) => {+ // Double-check: another instance may have authenticated while we waited for the lock+ try {+ const cached = await redis.get(REDIS_TOKEN_KEY);+ if (cached) {+ return cached;+ }+ } catch {}++ const username = config.WIKIPEDIA_ENTERPRISE_USERNAME;+ const password = config.WIKIPEDIA_ENTERPRISE_PASSWORD;++ if (!username || !password) {+ throw new EngineError(+ "Wikipedia Enterprise API credentials are not configured (WIKIPEDIA_ENTERPRISE_USERNAME, WIKIPEDIA_ENTERPRISE_PASSWORD)",+ );+ }++ logger.info("Authenticating with Wikipedia Enterprise API");++ const response = await undici.fetch(WIKIMEDIA_AUTH_URL, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({ username, password }),+ });++ if (signal.aborted) {+ throw signal.error;+ }++ if (!response.ok) {+ const body = await response.text();+ throw new EngineError(+ `Wikipedia Enterprise authentication failed (${response.status}): ${body}`,+ );+ }++ const data = (await response.json()) as {+ access_token: string;+ expires_in: number;+ };++ const ttlSeconds = Math.max(data.expires_in - 300, 60);++ try {+ await redis.set(REDIS_TOKEN_KEY, data.access_token, "EX", ttlSeconds);+ } catch (error) {+ logger.warn("Failed to cache Wikipedia token in Redis", { error });+ }++ return data.access_token;+ },+ );+}++function clearCachedToken(logger: Meta["logger"]): void {+ try {+ getRedisConnection().del(REDIS_TOKEN_KEY).catch((error) => {+ logger.warn("Failed to clear Wikipedia token from Redis", { error });+ });+ } catch {+ logger.warn("Failed to clear Wikipedia token from Redis");+ }+}++// Maps Wikimedia project hostnames to project identifiers used by the Enterprise API.+// e.g. "en.wikipedia.org" β { lang: "en", project: "wikipedia" }+export function parseWikimediaUrl(url: string): {+ articleName: string;+ lang: string;+ projectIdentifier: string;+} | null {+ let parsed: URL;+ try {+ parsed = new URL(url);+ } catch {+ return null;+ }++ const hostname = parsed.hostname;++ const wikiProjects: Record<string, string> = {+ "wikipedia.org": "wiki",+ "wiktionary.org": "wiktionary",+ "wikisource.org": "wikisource",+ "wikibooks.org": "wikibooks",+ "wikiquote.org": "wikiquote",+ "wikiversity.org": "wikiversity",+ "wikivoyage.org": "wikivoyage",+ };++ for (const [domain, projectSuffix] of Object.entries(wikiProjects)) {+ const regex = new RegExp(`^([a-z]{2,3})\\.${domain.replace(".", "\\.")}$`);+ const match = hostname.match(regex);+ if (match) {+ const lang = match[1];+ const pathMatch = parsed.pathname.match(/^\/wiki\/(.+)$/);+ if (!pathMatch) return null;++ return {+ articleName: decodeURIComponent(pathMatch[1]).replace(/ /g, "_"),+ lang,+ projectIdentifier: `${lang}${projectSuffix}`,+ };+ }+ }++ return null;+}++/**+ * Resolves redirects using Wikipedia's free MediaWiki API.+ * e.g. "Brasil" β "Brazil", "UK" β "United Kingdom"+ */+async function resolveRedirect(+ articleName: string,+ lang: string,+ domain: string,+ logger: Meta["logger"],+): Promise<string> {+ const apiUrl = `https://${lang}.${domain}/w/api.php?` ++ new URLSearchParams({+ action: "query",+ titles: articleName,+ redirects: "1",+ format: "json",+ formatversion: "2",+ }).toString();++ try {+ const response = await undici.fetch(apiUrl, {+ headers: { Accept: "application/json" },+ });++ if (!response.ok) return articleName;++ const data = (await response.json()) as {+ query?: {+ redirects?: { from: string; to: string }[];+ normalized?: { from: string; to: string }[];+ pages?: { title: string; missing?: boolean }[];+ };+ };++ const redirects = data.query?.redirects;+ if (redirects && redirects.length > 0) {+ const resolved = redirects[redirects.length - 1].to;+ logger.info("Resolved Wikipedia redirect", {+ from: articleName,+ to: resolved,+ });+ return resolved.replace(/ /g, "_");+ }++ const normalized = data.query?.normalized;+ if (normalized && normalized.length > 0) {+ return normalized[normalized.length - 1].to.replace(/ /g, "_");+ }++ return articleName;+ } catch (error) {+ logger.warn("Failed to resolve Wikipedia redirect, using original name", {+ articleName,+ error,+ });+ return articleName;+ }+}++interface WikimediaArticle {+ name: string;+ abstract: string;+ article_body: {+ html: string;+ wikitext: string;+ };+ url: string;+ in_language: {+ identifier: string;+ name: string;+ };+ is_part_of: {+ identifier: string;+ name: string;+ };+ date_modified: string;+ categories?: { name: string; url: string }[];+ has_parts?: { name: string }[];+}++export async function scrapeURLWithWikipedia(+ meta: Meta,+): Promise<EngineScrapeResult> {+ const urlToScrape = meta.rewrittenUrl ?? meta.url;+ const wikiInfo = parseWikimediaUrl(urlToScrape);++ if (!wikiInfo) {+ throw new EngineError(+ `URL is not a supported Wikimedia URL: ${urlToScrape}`,+ );+ }++ const { articleName: rawArticleName, lang, projectIdentifier } = wikiInfo;++ const domain = new URL(urlToScrape).hostname.split(".").slice(1).join(".");+ const articleName = await resolveRedirect(+ rawArticleName,+ lang,+ domain,+ meta.logger,+ );++ meta.logger.info("Fetching from Wikipedia Enterprise API", {+ articleName,+ rawArticleName,+ lang,+ projectIdentifier,+ });++ const token = await getAccessToken(meta.logger);++ const apiUrl = `${WIKIMEDIA_API_BASE}/articles/${encodeURIComponent(articleName)}`;++ const response = await undici.fetch(apiUrl, {+ method: "POST",+ headers: {+ Authorization: `Bearer ${token}`,+ "Content-Type": "application/json",+ Accept: "application/json",+ },+ body: JSON.stringify({+ filters: [+ {+ field: "is_part_of.identifier",+ value: projectIdentifier,+ },+ ],+ limit: 1,+ }),+ });++ if (response.status === 404) {+ throw new EngineError(+ `Wikipedia article not found: ${articleName} (${projectIdentifier})`,+ );+ }++ if (response.status === 401 || response.status === 403) {+ clearCachedToken(meta.logger);+ throw new EngineError(+ `Wikipedia Enterprise API authorization failed (${response.status}). Check credentials.`,+ );+ }++ if (!response.ok) {+ const body = await response.text();+ throw new EngineError(+ `Wikipedia Enterprise API error (${response.status}): ${body}`,+ );+ }++ const articles = (await response.json()) as WikimediaArticle[];++ if (!articles || articles.length === 0) {+ throw new EngineError(+ `No article data returned for: ${articleName} (${projectIdentifier})`,+ );+ }++ const article = articles[0];+ const html = article.article_body?.html;++ if (!html) {+ throw new EngineError(+ `Wikipedia article has no HTML body: ${articleName}`,+ );+ }++ // Build a full HTML document wrapping the article content with metadata+ const categories = (article.categories ?? [])+ .map(c => `<li><a href="${c.url}">${c.name}</a></li>`)+ .join("");+ const toc = (article.has_parts ?? [])+ .map(p => `<li>${p.name}</li>`)+ .join("");++ const fullHtml = `<!DOCTYPE html>+<html lang="${lang}">+<head>+ <meta charset="utf-8">+ <title>${article.name} - ${article.is_part_of?.name ?? "Wikipedia"}</title>+ <meta name="description" content="${escapeHtml(article.abstract ?? "")}">+ <meta property="og:title" content="${escapeHtml(article.name)}">+ <meta property="og:description" content="${escapeHtml(article.abstract ?? "")}">+ <meta property="og:url" content="${escapeHtml(article.url ?? urlToScrape)}">+ <meta name="article:modified_time" content="${article.date_modified ?? ""}">+ <meta name="language" content="${article.in_language?.name ?? lang}">+</head>+<body>+ <article>+ <h1>${escapeHtml(article.name)}</h1>+ ${html}+ </article>+ ${toc ? `<nav><h2>Table of Contents</h2><ul>${toc}</ul></nav>` : ""}+ ${categories ? `<footer><h2>Categories</h2><ul>${categories}</ul></footer>` : ""}+</body>+</html>`;++ meta.logger.info("Wikipedia Enterprise API article fetched successfully", {+ articleName: article.name,+ language: article.in_language?.identifier,+ project: article.is_part_of?.identifier,+ });++ return {+ url: article.url ?? urlToScrape,+ html: fullHtml,+ statusCode: 200,+ proxyUsed: "basic",+ };+}++function escapeHtml(str: string): string {+ return str+ .replace(/&/g, "&")+ .replace(/</g, "<")+ .replace(/>/g, ">")+ .replace(/"/g, """);+}++export function wikipediaMaxReasonableTime(_meta: Meta): number {+ return 10000;+}++export function isWikimediaUrl(url: string): boolean {+ return parseWikimediaUrl(url) !== null;+}