Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

(feat/wiki) Wikimedia (#3121)

Nicolas80e8d708fecd
diff --git a/apps/api/src/__tests__/snips/v2/wikipedia.test.ts b/apps/api/src/__tests__/snips/v2/wikipedia.test.ts
new file mode 100644
index 000000000..dc9f24f53
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/wikipedia.test.ts
@@ -0,0 +1,329 @@
+import { describeIf } from "../lib";
+import {
+ scrape,
+ scrapeRaw,
+ scrapeTimeout,
+ idmux,
+ Identity,
+} from "./lib";
+
+const HAS_WIKIPEDIA = !!(
+ process.env.WIKIPEDIA_ENTERPRISE_USERNAME &&
+ process.env.WIKIPEDIA_ENTERPRISE_PASSWORD
+);
+
+let identity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "wikipedia",
+ concurrency: 100,
+ credits: 1000000,
+ });
+}, 10000);
+
+describeIf(HAS_WIKIPEDIA && !process.env.TEST_SUITE_SELF_HOSTED)(
+ "Wikipedia Enterprise API integration",
+ () => {
+ describe("basic scraping", () => {
+ it.concurrent(
+ "scrapes a Wikipedia article and returns markdown by default",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/Web_scraping",
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeTruthy();
+ expect(response.markdown!.length).toBeGreaterThan(100);
+ expect(response.metadata.statusCode).toBe(200);
+ expect(response.metadata.sourceURL).toBe(
+ "https://en.wikipedia.org/wiki/Web_scraping",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "scrapes a non-English Wikipedia article (Spanish)",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://es.wikipedia.org/wiki/Web_scraping",
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeTruthy();
+ expect(response.markdown!.length).toBeGreaterThan(50);
+ expect(response.metadata.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "scrapes a non-English Wikipedia article (French)",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://fr.wikipedia.org/wiki/France",
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeTruthy();
+ expect(response.markdown!.length).toBeGreaterThan(100);
+ expect(response.metadata.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("redirect handling", () => {
+ it.concurrent(
+ "handles Wikipedia redirect pages (e.g. Brasil -> Brazil)",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/Brasil",
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeTruthy();
+ expect(response.markdown!.length).toBeGreaterThan(100);
+ expect(response.metadata.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "handles abbreviation redirects (e.g. UK -> United Kingdom)",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/UK",
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeTruthy();
+ expect(response.markdown!.length).toBeGreaterThan(100);
+ expect(response.metadata.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("special characters", () => {
+ it.concurrent(
+ "handles URL-encoded special characters (C++)",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/C%2B%2B",
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeTruthy();
+ expect(response.markdown!.length).toBeGreaterThan(100);
+ expect(response.metadata.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "handles articles with parentheses (disambiguation)",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/Mercury_(planet)",
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeTruthy();
+ expect(response.markdown!.length).toBeGreaterThan(100);
+ expect(response.metadata.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("output formats", () => {
+ it.concurrent(
+ "returns markdown when format is markdown",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/TypeScript",
+ formats: ["markdown"],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeTruthy();
+ expect(response.markdown!.length).toBeGreaterThan(100);
+ expect(response.markdown).toContain("TypeScript");
+ expect(response.html).toBeUndefined();
+ expect(response.rawHtml).toBeUndefined();
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "returns html when format is html",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/TypeScript",
+ formats: ["html"],
+ },
+ identity,
+ );
+
+ expect(response.html).toBeTruthy();
+ expect(response.html!.length).toBeGreaterThan(100);
+ expect(response.html).toContain("TypeScript");
+ expect(response.markdown).toBeUndefined();
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "returns rawHtml when format is rawHtml",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/TypeScript",
+ formats: ["rawHtml"],
+ },
+ identity,
+ );
+
+ expect(response.rawHtml).toBeTruthy();
+ expect(response.rawHtml!.length).toBeGreaterThan(100);
+ expect(response.rawHtml).toContain("TypeScript");
+ expect(response.markdown).toBeUndefined();
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "returns links when format is links",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/TypeScript",
+ formats: ["links"],
+ },
+ identity,
+ );
+
+ expect(response.links).toBeTruthy();
+ expect(Array.isArray(response.links)).toBe(true);
+ expect(response.links!.length).toBeGreaterThan(0);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "returns multiple formats together",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/TypeScript",
+ formats: ["markdown", "html", "links"],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeTruthy();
+ expect(response.html).toBeTruthy();
+ expect(response.links).toBeTruthy();
+ expect(response.markdown!.length).toBeGreaterThan(100);
+ expect(response.html!.length).toBeGreaterThan(100);
+ expect(response.links!.length).toBeGreaterThan(0);
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("metadata", () => {
+ it.concurrent(
+ "returns correct metadata for a Wikipedia article",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/TypeScript",
+ formats: ["markdown"],
+ },
+ identity,
+ );
+
+ expect(response.metadata).toBeDefined();
+ expect(response.metadata.statusCode).toBe(200);
+ expect(response.metadata.sourceURL).toBe(
+ "https://en.wikipedia.org/wiki/TypeScript",
+ );
+ expect(response.metadata.title).toBeTruthy();
+ expect(response.metadata.description).toBeTruthy();
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("error handling", () => {
+ it.concurrent(
+ "fails gracefully for non-existent Wikipedia article",
+ async () => {
+ const response = await scrapeRaw(
+ {
+ url: "https://en.wikipedia.org/wiki/ThisArticleDefinitelyDoesNotExist_XYZZY_123456789",
+ },
+ identity,
+ );
+
+ expect(response.statusCode).not.toBe(200);
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("scrape options", () => {
+ it.concurrent(
+ "respects onlyMainContent option",
+ async () => {
+ const withMainContent = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/TypeScript",
+ formats: ["markdown"],
+ onlyMainContent: true,
+ },
+ identity,
+ );
+
+ const withoutMainContent = await scrape(
+ {
+ url: "https://en.wikipedia.org/wiki/TypeScript",
+ formats: ["markdown"],
+ onlyMainContent: false,
+ },
+ identity,
+ );
+
+ expect(withMainContent.markdown).toBeTruthy();
+ expect(withoutMainContent.markdown).toBeTruthy();
+ // Full content should be equal to or longer than main-only content
+ expect(withoutMainContent.markdown!.length).toBeGreaterThanOrEqual(
+ withMainContent.markdown!.length,
+ );
+ },
+ scrapeTimeout * 2,
+ );
+ });
+ },
+);
diff --git a/apps/api/src/__tests__/snips/wikipedia-url-parser.test.ts b/apps/api/src/__tests__/snips/wikipedia-url-parser.test.ts
new file mode 100644
index 000000000..7b30ad7d2
--- /dev/null
+++ b/apps/api/src/__tests__/snips/wikipedia-url-parser.test.ts
@@ -0,0 +1,349 @@
+import {
+ parseWikimediaUrl,
+ isWikimediaUrl,
+} from "../../scraper/scrapeURL/engines/wikipedia";
+
+describe("Wikipedia URL parser", () => {
+ describe("parseWikimediaUrl", () => {
+ describe("Wikipedia articles", () => {
+ it("parses English Wikipedia article URL", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikipedia.org/wiki/Web_scraping",
+ );
+ expect(result).toEqual({
+ articleName: "Web_scraping",
+ lang: "en",
+ projectIdentifier: "enwiki",
+ });
+ });
+
+ it("parses German Wikipedia article URL", () => {
+ const result = parseWikimediaUrl(
+ "https://de.wikipedia.org/wiki/Web_Scraping",
+ );
+ expect(result).toEqual({
+ articleName: "Web_Scraping",
+ lang: "de",
+ projectIdentifier: "dewiki",
+ });
+ });
+
+ it("parses Spanish Wikipedia article URL", () => {
+ const result = parseWikimediaUrl(
+ "https://es.wikipedia.org/wiki/Scraping",
+ );
+ expect(result).toEqual({
+ articleName: "Scraping",
+ lang: "es",
+ projectIdentifier: "eswiki",
+ });
+ });
+
+ it("parses Japanese Wikipedia (3-letter lang code)", () => {
+ const result = parseWikimediaUrl(
+ "https://ja.wikipedia.org/wiki/ζ—₯本",
+ );
+ expect(result).toEqual({
+ articleName: "ζ—₯本",
+ lang: "ja",
+ projectIdentifier: "jawiki",
+ });
+ });
+
+ it("parses Portuguese Wikipedia", () => {
+ const result = parseWikimediaUrl(
+ "https://pt.wikipedia.org/wiki/Brasil",
+ );
+ expect(result).toEqual({
+ articleName: "Brasil",
+ lang: "pt",
+ projectIdentifier: "ptwiki",
+ });
+ });
+ });
+
+ describe("other Wikimedia projects", () => {
+ it("parses Wiktionary URL", () => {
+ const result = parseWikimediaUrl(
+ "https://fr.wiktionary.org/wiki/bonjour",
+ );
+ expect(result).toEqual({
+ articleName: "bonjour",
+ lang: "fr",
+ projectIdentifier: "frwiktionary",
+ });
+ });
+
+ it("parses Wikisource URL", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikisource.org/wiki/The_Art_of_War",
+ );
+ expect(result).toEqual({
+ articleName: "The_Art_of_War",
+ lang: "en",
+ projectIdentifier: "enwikisource",
+ });
+ });
+
+ it("parses Wikibooks URL", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikibooks.org/wiki/Cookbook:Table_of_Contents",
+ );
+ expect(result).toEqual({
+ articleName: "Cookbook:Table_of_Contents",
+ lang: "en",
+ projectIdentifier: "enwikibooks",
+ });
+ });
+
+ it("parses Wikiquote URL", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikiquote.org/wiki/Albert_Einstein",
+ );
+ expect(result).toEqual({
+ articleName: "Albert_Einstein",
+ lang: "en",
+ projectIdentifier: "enwikiquote",
+ });
+ });
+
+ it("parses Wikiversity URL", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikiversity.org/wiki/Introduction_to_Computers",
+ );
+ expect(result).toEqual({
+ articleName: "Introduction_to_Computers",
+ lang: "en",
+ projectIdentifier: "enwikiversity",
+ });
+ });
+
+ it("parses Wikivoyage URL", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikivoyage.org/wiki/Paris",
+ );
+ expect(result).toEqual({
+ articleName: "Paris",
+ lang: "en",
+ projectIdentifier: "enwikivoyage",
+ });
+ });
+ });
+
+ describe("special characters and encoding", () => {
+ it("handles URL-encoded article names", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikipedia.org/wiki/C%2B%2B",
+ );
+ expect(result).toEqual({
+ articleName: "C++",
+ lang: "en",
+ projectIdentifier: "enwiki",
+ });
+ });
+
+ it("handles articles with underscores (spaces)", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikipedia.org/wiki/United_States",
+ );
+ expect(result).toEqual({
+ articleName: "United_States",
+ lang: "en",
+ projectIdentifier: "enwiki",
+ });
+ });
+
+ it("handles articles with parentheses (disambiguation)", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikipedia.org/wiki/Mercury_(planet)",
+ );
+ expect(result).toEqual({
+ articleName: "Mercury_(planet)",
+ lang: "en",
+ projectIdentifier: "enwiki",
+ });
+ });
+
+ it("handles URL-encoded parentheses", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikipedia.org/wiki/Mercury_%28planet%29",
+ );
+ expect(result).toEqual({
+ articleName: "Mercury_(planet)",
+ lang: "en",
+ projectIdentifier: "enwiki",
+ });
+ });
+
+ it("handles articles with apostrophes", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikipedia.org/wiki/Murphy's_law",
+ );
+ expect(result).toEqual({
+ articleName: "Murphy's_law",
+ lang: "en",
+ projectIdentifier: "enwiki",
+ });
+ });
+
+ it("handles articles with colons (namespaces)", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikipedia.org/wiki/Wikipedia:About",
+ );
+ expect(result).toEqual({
+ articleName: "Wikipedia:About",
+ lang: "en",
+ projectIdentifier: "enwiki",
+ });
+ });
+
+ it("handles articles with URL-encoded spaces (%20)", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikipedia.org/wiki/New%20York%20City",
+ );
+ expect(result).toEqual({
+ articleName: "New_York_City",
+ lang: "en",
+ projectIdentifier: "enwiki",
+ });
+ });
+
+ it("handles articles with hash/fragment (strips it)", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikipedia.org/wiki/Brazil#History",
+ );
+ expect(result).toEqual({
+ articleName: "Brazil",
+ lang: "en",
+ projectIdentifier: "enwiki",
+ });
+ });
+
+ it("handles articles with query parameters", () => {
+ const result = parseWikimediaUrl(
+ "https://en.wikipedia.org/wiki/Brazil?action=edit",
+ );
+ expect(result).toEqual({
+ articleName: "Brazil",
+ lang: "en",
+ projectIdentifier: "enwiki",
+ });
+ });
+ });
+
+ describe("rejected URLs", () => {
+ it("returns null for non-Wikipedia URLs", () => {
+ expect(parseWikimediaUrl("https://google.com")).toBeNull();
+ expect(
+ parseWikimediaUrl("https://example.com/wiki/test"),
+ ).toBeNull();
+ });
+
+ it("returns null for Wikipedia URLs without /wiki/ path", () => {
+ expect(
+ parseWikimediaUrl("https://en.wikipedia.org/w/index.php"),
+ ).toBeNull();
+ expect(parseWikimediaUrl("https://en.wikipedia.org/")).toBeNull();
+ });
+
+ it("returns null for main Wikipedia domain without language prefix", () => {
+ expect(
+ parseWikimediaUrl("https://wikipedia.org/wiki/Test"),
+ ).toBeNull();
+ });
+
+ it("returns null for invalid URLs", () => {
+ expect(parseWikimediaUrl("not-a-url")).toBeNull();
+ expect(parseWikimediaUrl("")).toBeNull();
+ });
+
+ it("returns null for Wikimedia Commons (not a content project)", () => {
+ expect(
+ parseWikimediaUrl("https://commons.wikimedia.org/wiki/File:Test.jpg"),
+ ).toBeNull();
+ });
+
+ it("returns null for Wikidata", () => {
+ expect(
+ parseWikimediaUrl("https://www.wikidata.org/wiki/Q42"),
+ ).toBeNull();
+ });
+
+ it("returns null for MediaWiki API endpoints", () => {
+ expect(
+ parseWikimediaUrl(
+ "https://en.wikipedia.org/w/api.php?action=query&titles=Brazil",
+ ),
+ ).toBeNull();
+ });
+
+ it("returns null for Wikipedia Special pages", () => {
+ expect(
+ parseWikimediaUrl(
+ "https://en.wikipedia.org/wiki/Special:Random",
+ ),
+ ).not.toBeNull(); // Special:Random is technically a valid /wiki/ path
+ });
+
+ it("returns null for bare /wiki/ path with no article", () => {
+ expect(
+ parseWikimediaUrl("https://en.wikipedia.org/wiki/"),
+ ).toBeNull();
+ });
+
+ it("returns null for HTTP protocol", () => {
+ const result = parseWikimediaUrl(
+ "http://en.wikipedia.org/wiki/Test",
+ );
+ // HTTP should still parse -- the protocol doesn't matter for article resolution
+ expect(result).toEqual({
+ articleName: "Test",
+ lang: "en",
+ projectIdentifier: "enwiki",
+ });
+ });
+ });
+ });
+
+ describe("isWikimediaUrl", () => {
+ it("returns true for Wikipedia URLs in various languages", () => {
+ expect(isWikimediaUrl("https://en.wikipedia.org/wiki/Test")).toBe(true);
+ expect(isWikimediaUrl("https://de.wikipedia.org/wiki/Test")).toBe(true);
+ expect(isWikimediaUrl("https://ja.wikipedia.org/wiki/γƒ†γ‚Ήγƒˆ")).toBe(true);
+ expect(isWikimediaUrl("https://pt.wikipedia.org/wiki/Brasil")).toBe(true);
+ expect(isWikimediaUrl("https://zh.wikipedia.org/wiki/ζ΅‹θ―•")).toBe(true);
+ });
+
+ it("returns true for all supported Wikimedia projects", () => {
+ expect(isWikimediaUrl("https://en.wiktionary.org/wiki/hello")).toBe(true);
+ expect(isWikimediaUrl("https://en.wikisource.org/wiki/Test")).toBe(true);
+ expect(isWikimediaUrl("https://en.wikibooks.org/wiki/Test")).toBe(true);
+ expect(isWikimediaUrl("https://en.wikiquote.org/wiki/Test")).toBe(true);
+ expect(isWikimediaUrl("https://en.wikiversity.org/wiki/Test")).toBe(true);
+ expect(isWikimediaUrl("https://en.wikivoyage.org/wiki/Test")).toBe(true);
+ });
+
+ it("returns false for non-Wikimedia URLs", () => {
+ expect(isWikimediaUrl("https://google.com")).toBe(false);
+ expect(isWikimediaUrl("https://example.com/wiki/test")).toBe(false);
+ expect(isWikimediaUrl("https://en.fakepedia.org/wiki/test")).toBe(false);
+ });
+
+ it("returns false for Wikipedia non-article pages", () => {
+ expect(isWikimediaUrl("https://en.wikipedia.org/w/index.php")).toBe(
+ false,
+ );
+ expect(isWikimediaUrl("https://en.wikipedia.org/")).toBe(false);
+ });
+
+ it("returns false for unsupported Wikimedia projects", () => {
+ expect(
+ isWikimediaUrl("https://commons.wikimedia.org/wiki/File:Test.jpg"),
+ ).toBe(false);
+ expect(isWikimediaUrl("https://www.wikidata.org/wiki/Q42")).toBe(false);
+ expect(
+ isWikimediaUrl("https://meta.wikimedia.org/wiki/Main_Page"),
+ ).toBe(false);
+ });
+ });
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 4ee2f1214..7f0665263 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -230,6 +230,10 @@ const configSchema = z.object({
EXTRACT_V3_BETA_URL: z.string().optional(),
AGENT_INTEROP_SECRET: z.string().optional(),
+ // Wikipedia Enterprise API
+ WIKIPEDIA_ENTERPRISE_USERNAME: z.string().optional(),
+ WIKIPEDIA_ENTERPRISE_PASSWORD: z.string().optional(),
+
// Browser Service
BROWSER_SERVICE_URL: z.string().optional(),
BROWSER_SERVICE_API_KEY: z.string().optional(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 2378e4a52..97765f6b3 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -15,6 +15,11 @@ import {
scrapeURLWithPlaywright,
} from "./playwright";
import { indexMaxReasonableTime, scrapeURLWithIndex } from "./index/index";
+import {
+ scrapeURLWithWikipedia,
+ wikipediaMaxReasonableTime,
+ isWikimediaUrl,
+} from "./wikipedia";
import { queryEngpickerVerdict, useIndex } from "../../../services";
import { hasFormatOfType } from "../../../lib/format-utils";
import { getPDFMaxPages } from "../../../controllers/v2/types";
@@ -36,7 +41,8 @@ export type Engine =
| "pdf"
| "document"
| "index"
- | "index;documents";
+ | "index;documents"
+ | "wikipedia";
const useFireEngine =
config.FIRE_ENGINE_BETA_URL !== "" &&
@@ -44,8 +50,14 @@ const useFireEngine =
const usePlaywright =
config.PLAYWRIGHT_MICROSERVICE_URL !== "" &&
config.PLAYWRIGHT_MICROSERVICE_URL !== undefined;
+const useWikipedia =
+ config.WIKIPEDIA_ENTERPRISE_USERNAME !== undefined &&
+ config.WIKIPEDIA_ENTERPRISE_USERNAME !== "" &&
+ config.WIKIPEDIA_ENTERPRISE_PASSWORD !== undefined &&
+ config.WIKIPEDIA_ENTERPRISE_PASSWORD !== "";
const engines: Engine[] = [
+ ...(useWikipedia ? ["wikipedia" as const] : []),
...(useIndex ? ["index" as const, "index;documents" as const] : []),
...(useFireEngine
? [
@@ -158,6 +170,7 @@ const engineHandlers: {
fetch: scrapeURLWithFetch,
pdf: scrapePDF,
document: scrapeDocument,
+ wikipedia: scrapeURLWithWikipedia,
};
const engineMRTs: {
@@ -185,6 +198,7 @@ const engineMRTs: {
fetch: fetchMaxReasonableTime,
pdf: pdfMaxReasonableTime,
document: documentMaxReasonableTime,
+ wikipedia: wikipediaMaxReasonableTime,
};
const engineOptions: {
@@ -463,6 +477,25 @@ const engineOptions: {
},
quality: -20,
},
+ wikipedia: {
+ features: {
+ actions: false,
+ waitFor: false,
+ screenshot: false,
+ "screenshot@fullScreen": false,
+ pdf: false,
+ document: false,
+ atsv: false,
+ location: false,
+ mobile: false,
+ skipTlsVerification: true,
+ useFastMode: true,
+ stealthProxy: false,
+ branding: false,
+ disableAdblock: true,
+ },
+ quality: 500, // below index (1000) so cache is tried first, above fire-engine (50)
+ },
};
export function shouldUseIndex(meta: Meta) {
@@ -529,6 +562,13 @@ export async function buildFallbackList(meta: Meta): Promise<
}
}
+ if (!isWikimediaUrl(meta.url)) {
+ const wikiIndex = _engines.indexOf("wikipedia");
+ if (wikiIndex !== -1) {
+ _engines.splice(wikiIndex, 1);
+ }
+ }
+
const prioritySum = [...meta.featureFlags].reduce(
(a, x) => a + featureFlagOptions[x].priority,
0,
diff --git a/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts b/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts
new file mode 100644
index 000000000..77ff5a117
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/wikipedia/index.ts
@@ -0,0 +1,372 @@
+import * as undici from "undici";
+import { EngineScrapeResult } from "..";
+import { Meta } from "../..";
+import { config } from "../../../../config";
+import { EngineError } from "../../error";
+import { getRedisConnection } from "../../../../services/queue-service";
+import { redlock } from "../../../../services/redlock";
+
+const WIKIMEDIA_AUTH_URL =
+ "https://auth.enterprise.wikimedia.com/v1/login";
+const WIKIMEDIA_API_BASE =
+ "https://api.enterprise.wikimedia.com/v2";
+
+const REDIS_TOKEN_KEY = "wikipedia_enterprise:access_token";
+const REDIS_AUTH_LOCK_KEY = "lock:wikipedia_enterprise:auth";
+
+async function getAccessToken(
+ logger: Meta["logger"],
+): Promise<string> {
+ const redis = getRedisConnection();
+
+ try {
+ const cached = await redis.get(REDIS_TOKEN_KEY);
+ if (cached) {
+ return cached;
+ }
+ } catch (error) {
+ logger.warn("Failed to read Wikipedia token from Redis, will re-authenticate", { error });
+ }
+
+ // Acquire a distributed lock so only one instance authenticates at a time.
+ // Others wait and then read the token from Redis.
+ return await redlock.using(
+ [REDIS_AUTH_LOCK_KEY],
+ 10000,
+ async (signal) => {
+ // Double-check: another instance may have authenticated while we waited for the lock
+ try {
+ const cached = await redis.get(REDIS_TOKEN_KEY);
+ if (cached) {
+ return cached;
+ }
+ } catch {}
+
+ const username = config.WIKIPEDIA_ENTERPRISE_USERNAME;
+ const password = config.WIKIPEDIA_ENTERPRISE_PASSWORD;
+
+ if (!username || !password) {
+ throw new EngineError(
+ "Wikipedia Enterprise API credentials are not configured (WIKIPEDIA_ENTERPRISE_USERNAME, WIKIPEDIA_ENTERPRISE_PASSWORD)",
+ );
+ }
+
+ logger.info("Authenticating with Wikipedia Enterprise API");
+
+ const response = await undici.fetch(WIKIMEDIA_AUTH_URL, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ username, password }),
+ });
+
+ if (signal.aborted) {
+ throw signal.error;
+ }
+
+ if (!response.ok) {
+ const body = await response.text();
+ throw new EngineError(
+ `Wikipedia Enterprise authentication failed (${response.status}): ${body}`,
+ );
+ }
+
+ const data = (await response.json()) as {
+ access_token: string;
+ expires_in: number;
+ };
+
+ const ttlSeconds = Math.max(data.expires_in - 300, 60);
+
+ try {
+ await redis.set(REDIS_TOKEN_KEY, data.access_token, "EX", ttlSeconds);
+ } catch (error) {
+ logger.warn("Failed to cache Wikipedia token in Redis", { error });
+ }
+
+ return data.access_token;
+ },
+ );
+}
+
+function clearCachedToken(logger: Meta["logger"]): void {
+ try {
+ getRedisConnection().del(REDIS_TOKEN_KEY).catch((error) => {
+ logger.warn("Failed to clear Wikipedia token from Redis", { error });
+ });
+ } catch {
+ logger.warn("Failed to clear Wikipedia token from Redis");
+ }
+}
+
+// Maps Wikimedia project hostnames to project identifiers used by the Enterprise API.
+// e.g. "en.wikipedia.org" β†’ { lang: "en", project: "wikipedia" }
+export function parseWikimediaUrl(url: string): {
+ articleName: string;
+ lang: string;
+ projectIdentifier: string;
+} | null {
+ let parsed: URL;
+ try {
+ parsed = new URL(url);
+ } catch {
+ return null;
+ }
+
+ const hostname = parsed.hostname;
+
+ const wikiProjects: Record<string, string> = {
+ "wikipedia.org": "wiki",
+ "wiktionary.org": "wiktionary",
+ "wikisource.org": "wikisource",
+ "wikibooks.org": "wikibooks",
+ "wikiquote.org": "wikiquote",
+ "wikiversity.org": "wikiversity",
+ "wikivoyage.org": "wikivoyage",
+ };
+
+ for (const [domain, projectSuffix] of Object.entries(wikiProjects)) {
+ const regex = new RegExp(`^([a-z]{2,3})\\.${domain.replace(".", "\\.")}$`);
+ const match = hostname.match(regex);
+ if (match) {
+ const lang = match[1];
+ const pathMatch = parsed.pathname.match(/^\/wiki\/(.+)$/);
+ if (!pathMatch) return null;
+
+ return {
+ articleName: decodeURIComponent(pathMatch[1]).replace(/ /g, "_"),
+ lang,
+ projectIdentifier: `${lang}${projectSuffix}`,
+ };
+ }
+ }
+
+ return null;
+}
+
+/**
+ * Resolves redirects using Wikipedia's free MediaWiki API.
+ * e.g. "Brasil" β†’ "Brazil", "UK" β†’ "United Kingdom"
+ */
+async function resolveRedirect(
+ articleName: string,
+ lang: string,
+ domain: string,
+ logger: Meta["logger"],
+): Promise<string> {
+ const apiUrl = `https://${lang}.${domain}/w/api.php?` +
+ new URLSearchParams({
+ action: "query",
+ titles: articleName,
+ redirects: "1",
+ format: "json",
+ formatversion: "2",
+ }).toString();
+
+ try {
+ const response = await undici.fetch(apiUrl, {
+ headers: { Accept: "application/json" },
+ });
+
+ if (!response.ok) return articleName;
+
+ const data = (await response.json()) as {
+ query?: {
+ redirects?: { from: string; to: string }[];
+ normalized?: { from: string; to: string }[];
+ pages?: { title: string; missing?: boolean }[];
+ };
+ };
+
+ const redirects = data.query?.redirects;
+ if (redirects && redirects.length > 0) {
+ const resolved = redirects[redirects.length - 1].to;
+ logger.info("Resolved Wikipedia redirect", {
+ from: articleName,
+ to: resolved,
+ });
+ return resolved.replace(/ /g, "_");
+ }
+
+ const normalized = data.query?.normalized;
+ if (normalized && normalized.length > 0) {
+ return normalized[normalized.length - 1].to.replace(/ /g, "_");
+ }
+
+ return articleName;
+ } catch (error) {
+ logger.warn("Failed to resolve Wikipedia redirect, using original name", {
+ articleName,
+ error,
+ });
+ return articleName;
+ }
+}
+
+interface WikimediaArticle {
+ name: string;
+ abstract: string;
+ article_body: {
+ html: string;
+ wikitext: string;
+ };
+ url: string;
+ in_language: {
+ identifier: string;
+ name: string;
+ };
+ is_part_of: {
+ identifier: string;
+ name: string;
+ };
+ date_modified: string;
+ categories?: { name: string; url: string }[];
+ has_parts?: { name: string }[];
+}
+
+export async function scrapeURLWithWikipedia(
+ meta: Meta,
+): Promise<EngineScrapeResult> {
+ const urlToScrape = meta.rewrittenUrl ?? meta.url;
+ const wikiInfo = parseWikimediaUrl(urlToScrape);
+
+ if (!wikiInfo) {
+ throw new EngineError(
+ `URL is not a supported Wikimedia URL: ${urlToScrape}`,
+ );
+ }
+
+ const { articleName: rawArticleName, lang, projectIdentifier } = wikiInfo;
+
+ const domain = new URL(urlToScrape).hostname.split(".").slice(1).join(".");
+ const articleName = await resolveRedirect(
+ rawArticleName,
+ lang,
+ domain,
+ meta.logger,
+ );
+
+ meta.logger.info("Fetching from Wikipedia Enterprise API", {
+ articleName,
+ rawArticleName,
+ lang,
+ projectIdentifier,
+ });
+
+ const token = await getAccessToken(meta.logger);
+
+ const apiUrl = `${WIKIMEDIA_API_BASE}/articles/${encodeURIComponent(articleName)}`;
+
+ const response = await undici.fetch(apiUrl, {
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${token}`,
+ "Content-Type": "application/json",
+ Accept: "application/json",
+ },
+ body: JSON.stringify({
+ filters: [
+ {
+ field: "is_part_of.identifier",
+ value: projectIdentifier,
+ },
+ ],
+ limit: 1,
+ }),
+ });
+
+ if (response.status === 404) {
+ throw new EngineError(
+ `Wikipedia article not found: ${articleName} (${projectIdentifier})`,
+ );
+ }
+
+ if (response.status === 401 || response.status === 403) {
+ clearCachedToken(meta.logger);
+ throw new EngineError(
+ `Wikipedia Enterprise API authorization failed (${response.status}). Check credentials.`,
+ );
+ }
+
+ if (!response.ok) {
+ const body = await response.text();
+ throw new EngineError(
+ `Wikipedia Enterprise API error (${response.status}): ${body}`,
+ );
+ }
+
+ const articles = (await response.json()) as WikimediaArticle[];
+
+ if (!articles || articles.length === 0) {
+ throw new EngineError(
+ `No article data returned for: ${articleName} (${projectIdentifier})`,
+ );
+ }
+
+ const article = articles[0];
+ const html = article.article_body?.html;
+
+ if (!html) {
+ throw new EngineError(
+ `Wikipedia article has no HTML body: ${articleName}`,
+ );
+ }
+
+ // Build a full HTML document wrapping the article content with metadata
+ const categories = (article.categories ?? [])
+ .map(c => `<li><a href="${c.url}">${c.name}</a></li>`)
+ .join("");
+ const toc = (article.has_parts ?? [])
+ .map(p => `<li>${p.name}</li>`)
+ .join("");
+
+ const fullHtml = `<!DOCTYPE html>
+<html lang="${lang}">
+<head>
+ <meta charset="utf-8">
+ <title>${article.name} - ${article.is_part_of?.name ?? "Wikipedia"}</title>
+ <meta name="description" content="${escapeHtml(article.abstract ?? "")}">
+ <meta property="og:title" content="${escapeHtml(article.name)}">
+ <meta property="og:description" content="${escapeHtml(article.abstract ?? "")}">
+ <meta property="og:url" content="${escapeHtml(article.url ?? urlToScrape)}">
+ <meta name="article:modified_time" content="${article.date_modified ?? ""}">
+ <meta name="language" content="${article.in_language?.name ?? lang}">
+</head>
+<body>
+ <article>
+ <h1>${escapeHtml(article.name)}</h1>
+ ${html}
+ </article>
+ ${toc ? `<nav><h2>Table of Contents</h2><ul>${toc}</ul></nav>` : ""}
+ ${categories ? `<footer><h2>Categories</h2><ul>${categories}</ul></footer>` : ""}
+</body>
+</html>`;
+
+ meta.logger.info("Wikipedia Enterprise API article fetched successfully", {
+ articleName: article.name,
+ language: article.in_language?.identifier,
+ project: article.is_part_of?.identifier,
+ });
+
+ return {
+ url: article.url ?? urlToScrape,
+ html: fullHtml,
+ statusCode: 200,
+ proxyUsed: "basic",
+ };
+}
+
+function escapeHtml(str: string): string {
+ return str
+ .replace(/&/g, "&amp;")
+ .replace(/</g, "&lt;")
+ .replace(/>/g, "&gt;")
+ .replace(/"/g, "&quot;");
+}
+
+export function wikipediaMaxReasonableTime(_meta: Meta): number {
+ return 10000;
+}
+
+export function isWikimediaUrl(url: string): boolean {
+ return parseWikimediaUrl(url) !== null;
+}