Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
refactor(api): centralize firecrawl config (#2496)
tom166d067c8a6f
diff --git a/apps/api/src/__tests__/e2e_extract/index.test.ts b/apps/api/src/__tests__/e2e_extract/index.test.tsindex d1818643a..5547ff76c 100644--- a/apps/api/src/__tests__/e2e_extract/index.test.ts+++ b/apps/api/src/__tests__/e2e_extract/index.test.ts@@ -1,12 +1,11 @@import request from "supertest";-import dotenv from "dotenv";+import { config } from "../../config";import {FirecrawlCrawlResponse,FirecrawlCrawlStatusResponse,FirecrawlScrapeResponse,} from "../../types";-dotenv.config();const TEST_URL = "http://127.0.0.1:3002";describe("E2E Tests for Extract API Routes", () => {@@ -15,7 +14,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/extract")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({urls: ["https://firecrawl.dev/*"],@@ -53,7 +52,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/extract")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({urls: ["firecrawl.dev/*"],@@ -91,7 +90,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/extract")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({urls: ["https://firecrawl.dev/*"],@@ -128,7 +127,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/extract")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({urls: ["fivetran.com/*"],@@ -153,7 +152,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/extract")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({urls: ["fivetran.com/*"],@@ -184,7 +183,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/extract")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({urls: [@@ -214,7 +213,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/extract")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({urls: ["https://mintlify.com/docs/*"],@@ -258,7 +257,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/extract")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({urls: ["https://ericciarla.com/"],@@ -291,7 +290,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/extract")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({urls: ["https://docs.firecrawl.dev"],@@ -312,7 +311,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/extract")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({urls: [@@ -348,7 +347,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).get("/v2/extract/None")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).send();expect(response.statusCode).toBe(400);@@ -364,7 +363,7 @@ describe("E2E Tests for Extract API Routes", () => {async () => {const response = await request(TEST_URL).get("/v2/extract/not-a-uuid")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).send();expect(response.statusCode).toBe(400);diff --git a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_full_withAuth/index.test.tsindex 532b1d222..3b9bd5151 100644--- a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts+++ b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts@@ -1,20 +1,21 @@import request from "supertest";-import dotenv from "dotenv";+import { config } from "../../config";import { v7 as uuidv7 } from "uuid";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";-dotenv.config();-// const TEST_URL = 'http://localhost:3002'const TEST_URL = "http://127.0.0.1:3002";describe("E2E Tests for API Routes", () => {+ let originalUseDbAuth: boolean | undefined;+beforeAll(() => {- process.env.USE_DB_AUTHENTICATION = "true";+ originalUseDbAuth = config.USE_DB_AUTHENTICATION;+ config.USE_DB_AUTHENTICATION = true;});afterAll(() => {- delete process.env.USE_DB_AUTHENTICATION;+ config.USE_DB_AUTHENTICATION = originalUseDbAuth;});describe("GET /e2e-test", () => {@@ -47,7 +48,7 @@ describe("E2E Tests for API Routes", () => {const blocklistedUrl = "https://facebook.com/fake-test";const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: blocklistedUrl });expect(response.statusCode).toBe(403);@@ -59,7 +60,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://roastmywebsite.ai" });expect(response.statusCode).toBe(200);@@ -115,7 +116,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://roastmywebsite.ai",@@ -141,7 +142,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://roastmywebsite.ai",@@ -167,7 +168,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://arxiv.org/pdf/astro-ph/9301001.pdf" });await new Promise(r => setTimeout(r, 6000));@@ -190,7 +191,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://arxiv.org/pdf/astro-ph/9301001" });await new Promise(r => setTimeout(r, 6000));@@ -213,7 +214,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://arxiv.org/pdf/astro-ph/9301001.pdf",@@ -237,7 +238,7 @@ describe("E2E Tests for API Routes", () => {async () => {const responseWithoutRemoveTags = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://www.scrapethissite.com/" });expect(responseWithoutRemoveTags.statusCode).toBe(200);@@ -261,7 +262,7 @@ describe("E2E Tests for API Routes", () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://www.scrapethissite.com/",@@ -286,7 +287,7 @@ describe("E2E Tests for API Routes", () => {// const startTime = Date.now();// const response = await request(TEST_URL)// .post("/v0/scrape")- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)// .set("Content-Type", "application/json")// .send({ url: "https://firecrawl.dev", pageOptions: { waitFor: 7000 } });// const endTime = Date.now();@@ -307,7 +308,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/400" });await new Promise(r => setTimeout(r, 5000));@@ -329,7 +330,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/401" });await new Promise(r => setTimeout(r, 5000));@@ -351,7 +352,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/403" });@@ -373,7 +374,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/404" });await new Promise(r => setTimeout(r, 5000));@@ -395,7 +396,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/405" });await new Promise(r => setTimeout(r, 5000));@@ -417,7 +418,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/500" });await new Promise(r => setTimeout(r, 5000));@@ -457,7 +458,7 @@ describe("E2E Tests for API Routes", () => {const blocklistedUrl = "https://twitter.com/fake-test";const response = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: blocklistedUrl });expect(response.statusCode).toBe(403);@@ -469,7 +470,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://firecrawl.dev" });expect(response.statusCode).toBe(200);@@ -487,7 +488,7 @@ describe("E2E Tests for API Routes", () => {// First request with the idempotency keyconst firstResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").set("x-idempotency-key", uniqueIdempotencyKey).send({ url: "https://docs.firecrawl.dev" });@@ -497,7 +498,7 @@ describe("E2E Tests for API Routes", () => {// Second request with the same idempotency keyconst secondResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").set("x-idempotency-key", uniqueIdempotencyKey).send({ url: "https://docs.firecrawl.dev" });@@ -512,7 +513,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://mendable.ai",@@ -528,7 +529,7 @@ describe("E2E Tests for API Routes", () => {while (!isFinished) {response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -572,7 +573,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://mendable.ai",@@ -588,7 +589,7 @@ describe("E2E Tests for API Routes", () => {while (!isFinished) {response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -617,7 +618,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://mendable.ai",@@ -630,7 +631,7 @@ describe("E2E Tests for API Routes", () => {while (!isFinished) {response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -667,7 +668,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://www.scrapethissite.com",@@ -677,7 +678,7 @@ describe("E2E Tests for API Routes", () => {const response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");expect(["active", "waiting"]).toContain(response.body.status);@@ -686,7 +687,7 @@ describe("E2E Tests for API Routes", () => {while (!isCompleted) {const statusCheckResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(statusCheckResponse.statusCode).toBe(200);isCompleted = statusCheckResponse.body.status === "completed";if (!isCompleted) {@@ -695,7 +696,7 @@ describe("E2E Tests for API Routes", () => {}const completedResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(completedResponse.statusCode).toBe(200);expect(completedResponse.body).toHaveProperty("status");@@ -735,7 +736,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://www.scrapethissite.com/pages/",@@ -745,7 +746,7 @@ describe("E2E Tests for API Routes", () => {const response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");expect(["active", "waiting"]).toContain(response.body.status);@@ -754,7 +755,7 @@ describe("E2E Tests for API Routes", () => {while (!isCompleted) {const statusCheckResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(statusCheckResponse.statusCode).toBe(200);isCompleted = statusCheckResponse.body.status === "completed";if (!isCompleted) {@@ -763,7 +764,7 @@ describe("E2E Tests for API Routes", () => {}const completedResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(completedResponse.statusCode).toBe(200);expect(completedResponse.body).toHaveProperty("status");@@ -797,7 +798,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://www.mendable.ai",@@ -807,7 +808,7 @@ describe("E2E Tests for API Routes", () => {const response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");expect(["active", "waiting"]).toContain(response.body.status);@@ -816,7 +817,7 @@ describe("E2E Tests for API Routes", () => {while (!isCompleted) {const statusCheckResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(statusCheckResponse.statusCode).toBe(200);isCompleted = statusCheckResponse.body.status === "completed";if (!isCompleted) {@@ -825,7 +826,7 @@ describe("E2E Tests for API Routes", () => {}const completedResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);const testurls = completedResponse.body.data.map((item: any) => item.metadata?.sourceURL,@@ -862,7 +863,7 @@ describe("E2E Tests for API Routes", () => {// it.concurrent("should return a successful response with a valid API key and valid limit option", async () => {// const crawlResponse = await request(TEST_URL)// .post("/v0/crawl")- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)// .set("Content-Type", "application/json")// .send({// url: "https://mendable.ai",@@ -871,7 +872,7 @@ describe("E2E Tests for API Routes", () => {// const response = await request(TEST_URL)// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`);// expect(response.statusCode).toBe(200);// expect(response.body).toHaveProperty("status");// expect(response.body.status).toBe("active");@@ -880,7 +881,7 @@ describe("E2E Tests for API Routes", () => {// while (!isCompleted) {// const statusCheckResponse = await request(TEST_URL)// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`);// expect(statusCheckResponse.statusCode).toBe(200);// isCompleted = statusCheckResponse.body.status === "completed";// if (!isCompleted) {@@ -890,7 +891,7 @@ describe("E2E Tests for API Routes", () => {// const completedResponse = await request(TEST_URL)// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`);// expect(completedResponse.statusCode).toBe(200);// expect(completedResponse.body).toHaveProperty("status");@@ -909,7 +910,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://roastmywebsite.ai",@@ -919,7 +920,7 @@ describe("E2E Tests for API Routes", () => {const response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");expect(["active", "waiting"]).toContain(response.body.status);@@ -928,7 +929,7 @@ describe("E2E Tests for API Routes", () => {while (!isCompleted) {const statusCheckResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(statusCheckResponse.statusCode).toBe(200);isCompleted = statusCheckResponse.body.status === "completed";if (!isCompleted) {@@ -938,7 +939,7 @@ describe("E2E Tests for API Routes", () => {const completedResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(completedResponse.statusCode).toBe(200);expect(completedResponse.body).toHaveProperty("status");@@ -976,7 +977,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlInitResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://mendable.ai",@@ -996,7 +997,7 @@ describe("E2E Tests for API Routes", () => {while (crawlStatus !== "completed") {const statusResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlInitResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);crawlStatus = statusResponse.body.status;if (statusResponse.body.data) {crawlData = statusResponse.body.data;@@ -1048,7 +1049,7 @@ describe("E2E Tests for API Routes", () => {// const blocklistedUrl = "https://instagram.com/fake-test";// const response = await request(TEST_URL)// .post("/v0/crawlWebsitePreview")- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)// .set("Content-Type", "application/json")// .send({ url: blocklistedUrl });// // is returning 429 instead of 403@@ -1061,7 +1062,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://firecrawl.dev", timeout: 1000 });@@ -1094,7 +1095,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/search")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ query: "test" });expect(response.statusCode).toBe(200);@@ -1127,7 +1128,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).get("/v0/crawl/status/invalidJobId")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(404);},);@@ -1137,7 +1138,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://mendable.ai/blog" });expect(crawlResponse.statusCode).toBe(200);@@ -1148,7 +1149,7 @@ describe("E2E Tests for API Routes", () => {while (!isCompleted) {const response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -1190,7 +1191,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://arxiv.org/pdf/astro-ph/9301001",@@ -1214,7 +1215,7 @@ describe("E2E Tests for API Routes", () => {while (!isCompleted) {const response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -1254,7 +1255,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://roastmywebsite.ai",@@ -1264,7 +1265,7 @@ describe("E2E Tests for API Routes", () => {const response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");expect(["active", "waiting"]).toContain(response.body.status);@@ -1275,7 +1276,7 @@ describe("E2E Tests for API Routes", () => {while (!isFinished) {const response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -1314,7 +1315,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://mendable.ai/blog",@@ -1329,7 +1330,7 @@ describe("E2E Tests for API Routes", () => {while (!isFinished) {const response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -1374,7 +1375,7 @@ describe("E2E Tests for API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://jestjs.io" });@@ -1384,7 +1385,7 @@ describe("E2E Tests for API Routes", () => {const responseCancel = await request(TEST_URL).delete(`/v0/crawl/cancel/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(responseCancel.statusCode).toBe(200);expect(responseCancel.body).toHaveProperty("status");expect(responseCancel.body.status).toBe("cancelled");@@ -1392,7 +1393,7 @@ describe("E2E Tests for API Routes", () => {await new Promise(r => setTimeout(r, 10000));const completedResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(completedResponse.statusCode).toBe(200);expect(completedResponse.body).toHaveProperty("status");@@ -1419,7 +1420,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://mendable.ai",@@ -1472,7 +1473,7 @@ describe("E2E Tests for API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://mendable.ai",@@ -1516,7 +1517,7 @@ describe("E2E Tests for API Routes", () => {// it.concurrent("should extract data for the top 100 companies", async () => {// const response = await request(TEST_URL)// .post("/v0/scrape")- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)// .set("Content-Type", "application/json")// .send({// url: "https://companiesmarketcap.com/",@@ -1578,7 +1579,7 @@ describe("E2E Tests for API Routes", () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://flutterbricks.com",@@ -1596,7 +1597,7 @@ describe("E2E Tests for API Routes", () => {while (!isFinished) {statusResponse = await request(TEST_URL).get(`/v0/crawl/status/${jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(statusResponse.statusCode).toBe(200);isFinished = statusResponse.body.status === "completed";@@ -1634,7 +1635,7 @@ describe("E2E Tests for API Routes", () => {// const crawlResponse = await request(TEST_URL)// .post("/v0/crawl")- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)// .set("Content-Type", "application/json")// .send({// url: "https://flutterbricks.com",@@ -1649,7 +1650,7 @@ describe("E2E Tests for API Routes", () => {// while (!isFinished) {// statusResponse = await request(TEST_URL)// .get(`/v0/crawl/status/${jobId}`)- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`);// expect(statusResponse.statusCode).toBe(200);// isFinished = statusResponse.body.status === "completed";@@ -1693,7 +1694,7 @@ describe("E2E Tests for API Routes", () => {for (let i = 0; i < 5; i++) {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`)+ .set("Authorization", `Bearer ${config.PREVIEW_TOKEN}`).set("Content-Type", "application/json").send({ url: "https://www.scrapethissite.com" });@@ -1701,7 +1702,7 @@ describe("E2E Tests for API Routes", () => {}const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`)+ .set("Authorization", `Bearer ${config.PREVIEW_TOKEN}`).set("Content-Type", "application/json").send({ url: "https://www.scrapethissite.com" });@@ -1712,10 +1713,10 @@ describe("E2E Tests for API Routes", () => {});// it.concurrent("should return 429 when rate limit is exceeded for API key", async () => {- // for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) {+ // for (let i = 0; i < parseInt(config.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) {// const response = await request(TEST_URL)// .post("/v0/scrape")- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)// .set("Content-Type", "application/json")// .send({ url: "https://www.scrapethissite.com" });@@ -1724,7 +1725,7 @@ describe("E2E Tests for API Routes", () => {// const response = await request(TEST_URL)// .post("/v0/scrape")- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)// .set("Content-Type", "application/json")// .send({ url: "https://www.scrapethissite.com" });@@ -1732,10 +1733,10 @@ describe("E2E Tests for API Routes", () => {// }, 60000);// it.concurrent("should return 429 when rate limit is exceeded for API key", async () => {- // for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) {+ // for (let i = 0; i < parseInt(config.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) {// const response = await request(TEST_URL)// .post("/v0/crawl")- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)// .set("Content-Type", "application/json")// .send({ url: "https://www.scrapethissite.com" });@@ -1744,7 +1745,7 @@ describe("E2E Tests for API Routes", () => {// const response = await request(TEST_URL)// .post("/v0/crawl")- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)// .set("Content-Type", "application/json")// .send({ url: "https://www.scrapethissite.com" });diff --git a/apps/api/src/__tests__/e2e_map/index.test.ts b/apps/api/src/__tests__/e2e_map/index.test.tsindex 30ec6776e..26f25f101 100644--- a/apps/api/src/__tests__/e2e_map/index.test.ts+++ b/apps/api/src/__tests__/e2e_map/index.test.ts@@ -1,7 +1,6 @@import request from "supertest";-import dotenv from "dotenv";+import { config } from "../../config";-dotenv.config();const TEST_URL = "http://127.0.0.1:3002";describe("E2E Tests for Map API Routes", () => {@@ -10,7 +9,7 @@ describe("E2E Tests for Map API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://firecrawl.dev",@@ -32,7 +31,7 @@ describe("E2E Tests for Map API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://firecrawl.dev",@@ -56,7 +55,7 @@ describe("E2E Tests for Map API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://firecrawl.dev",@@ -79,7 +78,7 @@ describe("E2E Tests for Map API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://firecrawl.dev",@@ -100,7 +99,7 @@ describe("E2E Tests for Map API Routes", () => {async () => {const response = await request(TEST_URL).post("/v1/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://geekflare.com/sitemap_index.xml",diff --git a/apps/api/src/__tests__/e2e_map/v2_map.test.ts b/apps/api/src/__tests__/e2e_map/v2_map.test.tsindex 7d3987a80..ec255fa9d 100644--- a/apps/api/src/__tests__/e2e_map/v2_map.test.ts+++ b/apps/api/src/__tests__/e2e_map/v2_map.test.ts@@ -1,7 +1,6 @@import request from "supertest";-import dotenv from "dotenv";+import { config } from "../../config";-dotenv.config();const TEST_URL = "http://127.0.0.1:3002";describe("E2E Tests for Map V2 API Routes", () => {@@ -10,7 +9,7 @@ describe("E2E Tests for Map V2 API Routes", () => {async () => {const response = await request(TEST_URL).post("/v2/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://roastmywebsite.ai",@@ -46,7 +45,7 @@ describe("E2E Tests for Map V2 API Routes", () => {async () => {const response = await request(TEST_URL).post("/v2/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://firecrawl.dev",@@ -67,7 +66,7 @@ describe("E2E Tests for Map V2 API Routes", () => {async () => {const response = await request(TEST_URL).post("/v2/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://firecrawl.dev",@@ -87,7 +86,7 @@ describe("E2E Tests for Map V2 API Routes", () => {async () => {const response = await request(TEST_URL).post("/v2/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://firecrawl.dev",@@ -109,7 +108,7 @@ describe("E2E Tests for Map V2 API Routes", () => {async () => {const response = await request(TEST_URL).post("/v2/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://firecrawl.dev",diff --git a/apps/api/src/__tests__/e2e_noAuth/index.test.ts b/apps/api/src/__tests__/e2e_noAuth/index.test.tsindex 396521e0b..428202a85 100644--- a/apps/api/src/__tests__/e2e_noAuth/index.test.ts+++ b/apps/api/src/__tests__/e2e_noAuth/index.test.ts@@ -1,33 +1,41 @@import request from "supertest";-import dotenv from "dotenv";+import { config } from "../../config";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";const fs = require("fs");const path = require("path");-dotenv.config();-const TEST_URL = "http://127.0.0.1:3002";describe("E2E Tests for API Routes with No Authentication", () => {- let originalEnv: NodeJS.ProcessEnv;+ let originalConfig: Partial<typeof config>;- // save original process.env+ // save original config valuesbeforeAll(() => {- originalEnv = { ...process.env };- process.env.USE_DB_AUTHENTICATION = "false";- process.env.SUPABASE_ANON_TOKEN = "";- process.env.SUPABASE_URL = "";- process.env.SUPABASE_SERVICE_TOKEN = "";- process.env.OPENAI_API_KEY = "";- process.env.BULL_AUTH_KEY = "";- process.env.PLAYWRIGHT_MICROSERVICE_URL = "";- process.env.LLAMAPARSE_API_KEY = "";- process.env.TEST_API_KEY = "";+ originalConfig = {+ USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,+ SUPABASE_ANON_TOKEN: config.SUPABASE_ANON_TOKEN,+ SUPABASE_URL: config.SUPABASE_URL,+ SUPABASE_SERVICE_TOKEN: config.SUPABASE_SERVICE_TOKEN,+ OPENAI_API_KEY: config.OPENAI_API_KEY,+ BULL_AUTH_KEY: config.BULL_AUTH_KEY,+ PLAYWRIGHT_MICROSERVICE_URL: config.PLAYWRIGHT_MICROSERVICE_URL,+ LLAMAPARSE_API_KEY: config.LLAMAPARSE_API_KEY,+ TEST_API_KEY: config.TEST_API_KEY,+ };+ config.USE_DB_AUTHENTICATION = false;+ config.SUPABASE_ANON_TOKEN = "";+ config.SUPABASE_URL = "";+ config.SUPABASE_SERVICE_TOKEN = "";+ config.OPENAI_API_KEY = "";+ config.BULL_AUTH_KEY = "";+ config.PLAYWRIGHT_MICROSERVICE_URL = "";+ config.LLAMAPARSE_API_KEY = "";+ config.TEST_API_KEY = "";});- // restore original process.env+ // restore original config valuesafterAll(() => {- process.env = originalEnv;+ Object.assign(config, originalConfig);});describe("GET /e2e-test", () => {diff --git a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.tsindex b609050d8..ffb3eab49 100644--- a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts+++ b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts@@ -1,4 +1,5 @@import request from "supertest";+import { config } from "../../config";import { configDotenv } from "dotenv";import { ScrapeRequestInput } from "../../controllers/v1/types";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";@@ -8,23 +9,20 @@ const TEST_URL = "http://127.0.0.1:3002";describe("E2E Tests for v1 API Routes", () => {beforeAll(() => {- process.env.USE_DB_AUTHENTICATION = "true";+ config.USE_DB_AUTHENTICATION = true;});afterAll(() => {- delete process.env.USE_DB_AUTHENTICATION;+ delete config.USE_DB_AUTHENTICATION;});describe("GET /is-production", () => {it.concurrent("should return the production status", async () => {const response: any = await request(TEST_URL).get("/is-production");- console.log(- "process.env.USE_DB_AUTHENTICATION",- process.env.USE_DB_AUTHENTICATION,- );- console.log("?", process.env.USE_DB_AUTHENTICATION === "true");- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";+ console.log("config.USE_DB_AUTHENTICATION", config.USE_DB_AUTHENTICATION);+ console.log("?", config.USE_DB_AUTHENTICATION);+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;console.log("!!useDbAuthentication", !!useDbAuthentication);console.log("!useDbAuthentication", !useDbAuthentication);@@ -49,7 +47,7 @@ describe("E2E Tests for v1 API Routes", () => {const response = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -78,7 +76,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -130,7 +128,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -187,7 +185,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -215,7 +213,7 @@ describe("E2E Tests for v1 API Routes", () => {};const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);await new Promise(r => setTimeout(r, 6000));@@ -243,7 +241,7 @@ describe("E2E Tests for v1 API Routes", () => {};const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);await new Promise(r => setTimeout(r, 6000));@@ -273,7 +271,7 @@ describe("E2E Tests for v1 API Routes", () => {};const responseWithoutRemoveTags: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);expect(responseWithoutRemoveTags.statusCode).toBe(200);@@ -299,7 +297,7 @@ describe("E2E Tests for v1 API Routes", () => {};const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequestWithRemoveTags);@@ -322,7 +320,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/400" });await new Promise(r => setTimeout(r, 5000));@@ -344,7 +342,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/401" });await new Promise(r => setTimeout(r, 5000));@@ -365,7 +363,7 @@ describe("E2E Tests for v1 API Routes", () => {// it.concurrent('should return a successful response for a scrape with 403 page', async () => {// const response: any = await request(TEST_URL)// .post('/v1/scrape')- // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)+ // .set('Authorization', `Bearer ${config.TEST_API_KEY}`)// .set('Content-Type', 'application/json')// .send({ url: 'https://httpstat.us/403' });// await new Promise((r) => setTimeout(r, 5000));@@ -385,7 +383,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/404" });await new Promise(r => setTimeout(r, 5000));@@ -405,7 +403,7 @@ describe("E2E Tests for v1 API Routes", () => {// it.concurrent('should return a successful response for a scrape with 405 page', async () => {// const response: any = await request(TEST_URL)// .post('/v1/scrape')- // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)+ // .set('Authorization', `Bearer ${config.TEST_API_KEY}`)// .set('Content-Type', 'application/json')// .send({ url: 'https://httpstat.us/405' });// await new Promise((r) => setTimeout(r, 5000));@@ -423,7 +421,7 @@ describe("E2E Tests for v1 API Routes", () => {// it.concurrent('should return a successful response for a scrape with 500 page', async () => {// const response: any = await request(TEST_URL)// .post('/v1/scrape')- // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)+ // .set('Authorization', `Bearer ${config.TEST_API_KEY}`)// .set('Content-Type', 'application/json')// .send({ url: 'https://httpstat.us/500' });// await new Promise((r) => setTimeout(r, 5000));@@ -443,7 +441,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://firecrawl.dev", timeout: 1000 });@@ -462,7 +460,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -494,7 +492,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -525,7 +523,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -575,7 +573,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(mapRequest);@@ -601,7 +599,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(mapRequest);@@ -629,7 +627,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(mapRequest);@@ -661,7 +659,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(mapRequest);@@ -694,7 +692,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(mapRequest);@@ -720,7 +718,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(TEST_URL).post("/v1/map")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(mapRequest);@@ -745,7 +743,7 @@ describe("E2E Tests for v1 API Routes", () => {const response = await request(TEST_URL).post("/v1/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -768,7 +766,7 @@ describe("E2E Tests for v1 API Routes", () => {it.concurrent("should return a successful response", async () => {const response = await request(TEST_URL).post("/v1/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://firecrawl.dev" });@@ -787,7 +785,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v1/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://firecrawl.dev",@@ -801,7 +799,7 @@ describe("E2E Tests for v1 API Routes", () => {while (!isFinished) {response = await request(TEST_URL).get(`/v1/crawl/${crawlResponse.body.id}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -815,7 +813,7 @@ describe("E2E Tests for v1 API Routes", () => {await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the databaseconst completedResponse = await request(TEST_URL).get(`/v1/crawl/${crawlResponse.body.id}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);const urls = completedResponse.body.data.map((item: any) => item.metadata?.sourceURL,@@ -843,7 +841,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v1/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://firecrawl.dev",@@ -857,7 +855,7 @@ describe("E2E Tests for v1 API Routes", () => {while (!isFinished) {response = await request(TEST_URL).get(`/v1/crawl/${crawlResponse.body.id}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -871,7 +869,7 @@ describe("E2E Tests for v1 API Routes", () => {await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the databaseconst completedResponse = await request(TEST_URL).get(`/v1/crawl/${crawlResponse.body.id}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);const urls = completedResponse.body.data.map((item: any) => item.metadata?.sourceURL,@@ -889,7 +887,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v1/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://www.scrapethissite.com",@@ -899,7 +897,7 @@ describe("E2E Tests for v1 API Routes", () => {const response = await request(TEST_URL).get(`/v1/crawl/${crawlResponse.body.id}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");expect(["active", "waiting", "completed", "scraping"]).toContain(@@ -910,7 +908,7 @@ describe("E2E Tests for v1 API Routes", () => {while (!isCompleted) {const statusCheckResponse = await request(TEST_URL).get(`/v1/crawl/${crawlResponse.body.id}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(statusCheckResponse.statusCode).toBe(200);isCompleted = statusCheckResponse.body.status === "completed";if (!isCompleted) {@@ -919,7 +917,7 @@ describe("E2E Tests for v1 API Routes", () => {}const completedResponse = await request(TEST_URL).get(`/v1/crawl/${crawlResponse.body.id}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(completedResponse.statusCode).toBe(200);expect(completedResponse.body).toHaveProperty("status");@@ -972,7 +970,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const response = await request(TEST_URL).get("/v1/crawl/invalidJobId")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(404);},);@@ -982,7 +980,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v1/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://docs.firecrawl.dev" });expect(crawlResponse.statusCode).toBe(200);@@ -992,7 +990,7 @@ describe("E2E Tests for v1 API Routes", () => {while (!isCompleted) {const response = await request(TEST_URL).get(`/v1/crawl/${crawlResponse.body.id}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -1006,7 +1004,7 @@ describe("E2E Tests for v1 API Routes", () => {await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the databaseconst completedResponse = await request(TEST_URL).get(`/v1/crawl/${crawlResponse.body.id}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(completedResponse.body).toHaveProperty("status");expect(completedResponse.body.status).toBe("completed");@@ -1031,7 +1029,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v1/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://docs.firecrawl.dev", limit: 10 });@@ -1041,7 +1039,7 @@ describe("E2E Tests for v1 API Routes", () => {const responseCancel = await request(TEST_URL).delete(`/v1/crawl/${crawlResponse.body.id}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(responseCancel.statusCode).toBe(200);expect(responseCancel.body).toHaveProperty("status");expect(responseCancel.body.status).toBe("cancelled");@@ -1049,7 +1047,7 @@ describe("E2E Tests for v1 API Routes", () => {await new Promise(r => setTimeout(r, 10000));const completedResponse = await request(TEST_URL).get(`/v1/crawl/${crawlResponse.body.id}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(completedResponse.statusCode).toBe(200);expect(completedResponse.body).toHaveProperty("status");diff --git a/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.tsindex 5eb39f27a..268ece80b 100644--- a/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts+++ b/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts@@ -1,4 +1,5 @@import request from "supertest";+import { config } from "../../config";import { configDotenv } from "dotenv";import { ScrapeRequest } from "../../controllers/v1/types";@@ -12,7 +13,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/403" });@@ -37,7 +38,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -82,7 +83,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -115,7 +116,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -151,7 +152,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -178,7 +179,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -208,7 +209,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -239,7 +240,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -269,7 +270,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -294,7 +295,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -315,7 +316,7 @@ describe("E2E Tests for v1 API Routes", () => {async () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://arxiv.org/pdf/astro-ph/9301001.pdf" });await new Promise(r => setTimeout(r, 6000));@@ -335,7 +336,7 @@ describe("E2E Tests for v1 API Routes", () => {const responseNoParsePDF: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://arxiv.org/pdf/astro-ph/9301001.pdf",@@ -367,7 +368,7 @@ describe("E2E Tests for v1 API Routes", () => {// const response: any = await request(FIRECRAWL_API_URL)// .post("/v1/scrape")- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)// .set("Content-Type", "application/json")// .send(scrapeRequest);@@ -386,7 +387,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);console.log("Error1a");@@ -408,7 +409,7 @@ describe("E2E Tests for v1 API Routes", () => {FIRECRAWL_API_URL,).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequestWithSkipTlsVerification);@@ -436,7 +437,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -466,7 +467,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -497,7 +498,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -538,7 +539,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -586,7 +587,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -622,7 +623,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -655,7 +656,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);@@ -692,7 +693,7 @@ describe("E2E Tests for v1 API Routes", () => {const response: any = await request(FIRECRAWL_API_URL).post("/v1/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send(scrapeRequest);diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.tsindex e5a300092..8cb7e53df 100644--- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts+++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts@@ -1,16 +1,15 @@import request from "supertest";-import dotenv from "dotenv";+import { config } from "../../config";-dotenv.config();const TEST_URL = "http://127.0.0.1:3002";describe("E2E Tests for v0 API Routes", () => {beforeAll(() => {- process.env.USE_DB_AUTHENTICATION = "true";+ config.USE_DB_AUTHENTICATION = true;});afterAll(() => {- delete process.env.USE_DB_AUTHENTICATION;+ delete config.USE_DB_AUTHENTICATION;});describe("GET /is-production", () => {@@ -44,7 +43,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://roastmywebsite.ai" });expect(response.statusCode).toBe(200);@@ -88,7 +87,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://roastmywebsite.ai",@@ -114,7 +113,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://arxiv.org/pdf/astro-ph/9301001.pdf" });await new Promise(r => setTimeout(r, 6000));@@ -137,7 +136,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://arxiv.org/pdf/astro-ph/9301001" });await new Promise(r => setTimeout(r, 6000));@@ -160,7 +159,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const responseWithoutRemoveTags: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://www.scrapethissite.com/" });expect(responseWithoutRemoveTags.statusCode).toBe(200);@@ -184,7 +183,7 @@ describe("E2E Tests for v0 API Routes", () => {const response: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://www.scrapethissite.com/",@@ -209,7 +208,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/400" });await new Promise(r => setTimeout(r, 5000));@@ -231,7 +230,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/401" });await new Promise(r => setTimeout(r, 5000));@@ -253,7 +252,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/403" });@@ -275,7 +274,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/404" });await new Promise(r => setTimeout(r, 5000));@@ -294,7 +293,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/405" });await new Promise(r => setTimeout(r, 5000));@@ -313,7 +312,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://httpstat.us/500" });await new Promise(r => setTimeout(r, 5000));@@ -351,7 +350,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://firecrawl.dev" });expect(response.statusCode).toBe(200);@@ -367,7 +366,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const crawlResponse: any = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://mendable.ai",@@ -383,7 +382,7 @@ describe("E2E Tests for v0 API Routes", () => {while (!isFinished) {response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -397,7 +396,7 @@ describe("E2E Tests for v0 API Routes", () => {await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the databaseconst completedResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);const urls = completedResponse.body.data.map((item: any) => item.metadata?.sourceURL,@@ -430,7 +429,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const crawlResponse: any = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://mendable.ai",@@ -446,7 +445,7 @@ describe("E2E Tests for v0 API Routes", () => {while (!isFinished) {response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -460,7 +459,7 @@ describe("E2E Tests for v0 API Routes", () => {await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the databaseconst completedResponse: any = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);const urls = completedResponse.body.data.map((item: any) => item.metadata?.sourceURL,@@ -478,7 +477,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const crawlResponse: any = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://www.scrapethissite.com",@@ -488,7 +487,7 @@ describe("E2E Tests for v0 API Routes", () => {const response: any = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");expect(["active", "waiting"]).toContain(response.body.status);@@ -497,7 +496,7 @@ describe("E2E Tests for v0 API Routes", () => {while (!isCompleted) {const statusCheckResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(statusCheckResponse.statusCode).toBe(200);isCompleted = statusCheckResponse.body.status === "completed";if (!isCompleted) {@@ -506,7 +505,7 @@ describe("E2E Tests for v0 API Routes", () => {}const completedResponse: any = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(completedResponse.statusCode).toBe(200);expect(completedResponse.body).toHaveProperty("status");@@ -567,7 +566,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response: any = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://firecrawl.dev", timeout: 1000 });@@ -600,7 +599,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/search")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ query: "test" });expect(response.statusCode).toBe(200);@@ -633,7 +632,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response = await request(TEST_URL).get("/v0/crawl/status/invalidJobId")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(404);},);@@ -643,7 +642,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({ url: "https://firecrawl.dev/blog" });expect(crawlResponse.statusCode).toBe(200);@@ -653,7 +652,7 @@ describe("E2E Tests for v0 API Routes", () => {while (!isCompleted) {const response = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(response.statusCode).toBe(200);expect(response.body).toHaveProperty("status");@@ -667,7 +666,7 @@ describe("E2E Tests for v0 API Routes", () => {await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the databaseconst completedResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(completedResponse.body).toHaveProperty("status");expect(completedResponse.body.status).toBe("completed");@@ -699,7 +698,7 @@ describe("E2E Tests for v0 API Routes", () => {// it.concurrent('should return a successful response for a valid crawl job with PDF files without explicit .pdf extension ', async () => {// const crawlResponse = await request(TEST_URL)// .post('/v0/crawl')- // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)+ // .set('Authorization', `Bearer ${config.TEST_API_KEY}`)// .set('Content-Type', 'application/json')// .send({ url: 'https://arxiv.org/list/astro-ph/1993-01',// crawlerOptions: {@@ -714,7 +713,7 @@ describe("E2E Tests for v0 API Routes", () => {// while (!isCompleted) {// const response = await request(TEST_URL)// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`);+ // .set('Authorization', `Bearer ${config.TEST_API_KEY}`);// expect(response.statusCode).toBe(200);// expect(response.body).toHaveProperty('status');@@ -746,7 +745,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const crawlResponse = await request(TEST_URL).post("/v0/crawl")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://docs.tatum.io",@@ -759,7 +758,7 @@ describe("E2E Tests for v0 API Routes", () => {const responseCancel = await request(TEST_URL).delete(`/v0/crawl/cancel/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);expect(responseCancel.statusCode).toBe(200);expect(responseCancel.body).toHaveProperty("status");expect(responseCancel.body.status).toBe("cancelled");@@ -767,7 +766,7 @@ describe("E2E Tests for v0 API Routes", () => {await new Promise(r => setTimeout(r, 10000));const completedResponse = await request(TEST_URL).get(`/v0/crawl/status/${crawlResponse.body.jobId}`)- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).maxResponseSize(4000000000);expect(completedResponse.statusCode).toBe(200);@@ -811,7 +810,7 @@ describe("E2E Tests for v0 API Routes", () => {async () => {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`).set("Content-Type", "application/json").send({url: "https://mendable.ai",diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex 11b6f9632..a802fd8f2 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -1,4 +1,5 @@import { configDotenv } from "dotenv";+import { config } from "../../config";configDotenv();import { TeamFlags } from "../../controllers/v1/types";@@ -7,7 +8,7 @@ import { TeamFlags } from "../../controllers/v1/types";// Configuration// =========================================-export const TEST_API_URL = process.env.TEST_API_URL || "http://127.0.0.1:3002";+export const TEST_API_URL = config.TEST_API_URL;export const TEST_URL = TEST_API_URL; // backwards compat tempconst stripTrailingSlash = (url: string) => {@@ -15,21 +16,17 @@ const stripTrailingSlash = (url: string) => {return url.endsWith("/") ? url.substring(0, url.length - 1) : url;};-export const TEST_SUITE_WEBSITE = stripTrailingSlash(- process.env.TEST_SUITE_WEBSITE || "http://127.0.0.1:4321",-);+export const TEST_SUITE_WEBSITE = stripTrailingSlash(config.TEST_SUITE_WEBSITE);-export const TEST_SELF_HOST = process.env.TEST_SUITE_SELF_HOSTED === "true";+export const TEST_SELF_HOST = !!config.TEST_SUITE_SELF_HOSTED;export const TEST_PRODUCTION = !TEST_SELF_HOST;// TODO: do we want to run AI tests when users run this command locally? It may lead to increased spending for them, depending on configuration-export const HAS_AI = !!(- process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL-);-export const HAS_PLAYWRIGHT = !!process.env.PLAYWRIGHT_MICROSERVICE_URL;-export const HAS_PROXY = !!process.env.PROXY_SERVER;+export const HAS_AI = !!(config.OPENAI_API_KEY || config.OLLAMA_BASE_URL);+export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;+export const HAS_PROXY = !!config.PROXY_SERVER;-export const HAS_SEARCH = TEST_PRODUCTION || !!process.env.SEARXNG_ENDPOINT;+export const HAS_SEARCH = TEST_PRODUCTION || !!config.SEARXNG_ENDPOINT;const isLocalUrl = (x: string) =>/^https?:\/\/(localhost|127\.0\.0\.1|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})(:\d+)?([\/?#]|$)/i.test(@@ -52,7 +49,7 @@ export const createTestIdUrl = () =>if (isLocalUrl(TEST_SUITE_WEBSITE)) {if (TEST_SELF_HOST) {- process.env.ALLOW_LOCAL_WEBHOOKS = "true";+ config.ALLOW_LOCAL_WEBHOOKS = true;} else {throw new Error("TEST_SUITE_WEBSITE cannot be a local address while testing in production",@@ -79,25 +76,25 @@ export type IdmuxRequest = {};export async function idmux(req: IdmuxRequest): Promise<Identity> {- if (!process.env.IDMUX_URL) {+ if (!config.IDMUX_URL) {if (TEST_PRODUCTION) {console.warn("IDMUX_URL is not set, using test API key and team ID");}return {- apiKey: process.env.TEST_API_KEY!,- teamId: process.env.TEST_TEAM_ID!,+ apiKey: config.TEST_API_KEY!,+ teamId: config.TEST_TEAM_ID!,};}- let runNumber = parseInt(process.env.GITHUB_RUN_NUMBER!);+ let runNumber = parseInt(config.GITHUB_RUN_NUMBER!);if (isNaN(runNumber) || runNumber === null || runNumber === undefined) {runNumber = 0;}- const res = await fetch(process.env.IDMUX_URL + "/", {+ const res = await fetch(config.IDMUX_URL + "/", {method: "POST",body: JSON.stringify({- refName: process.env.GITHUB_REF_NAME!,+ refName: config.GITHUB_REF_NAME!,runNumber,concurrency: req.concurrency ?? 100,...req,diff --git a/apps/api/src/__tests__/snips/v1/lib.ts b/apps/api/src/__tests__/snips/v1/lib.tsindex 5664cb87f..9c905c466 100644--- a/apps/api/src/__tests__/snips/v1/lib.ts+++ b/apps/api/src/__tests__/snips/v1/lib.ts@@ -1,3 +1,4 @@+import { config } from "../../../config";import {ScrapeRequestInput,Document,@@ -548,7 +549,7 @@ export async function batchScrapeWithConcurrencyTracking(export async function zdrcleaner(teamId: string) {const res = await request(TEST_API_URL)- .get(`/admin/${process.env.BULL_AUTH_KEY}/zdrcleaner`)+ .get(`/admin/${config.BULL_AUTH_KEY}/zdrcleaner`).query({ teamId });expect(res.statusCode).toBe(200);diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.tsindex 99b30825f..31ded4a57 100644--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts@@ -1,3 +1,4 @@+import { config } from "../../../config";import {createTestIdUrl,describeIf,@@ -188,7 +189,7 @@ describe("Scrape tests", () => {);expect(response.markdown?.trim()).toContain(- process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],+ config.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],);},scrapeTimeout,@@ -208,7 +209,7 @@ describe("Scrape tests", () => {);expect(response.markdown?.trim()).toContain(- process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],+ config.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],);},scrapeTimeout,diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex 7dc175164..2a119a4c8 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -1,3 +1,4 @@+import { config } from "../../../config";import {ScrapeRequestInput,Document,@@ -547,7 +548,7 @@ export async function extractRaw(export async function zdrcleaner(teamId: string) {const res = await request(TEST_API_URL)- .get(`/admin/${process.env.BULL_AUTH_KEY}/zdrcleaner`)+ .get(`/admin/${config.BULL_AUTH_KEY}/zdrcleaner`).query({ teamId });expect(res.statusCode).toBe(200);diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 676b7f735..9719c92bf 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -1,3 +1,4 @@+import { config } from "../../../config";import {createTestIdUrl,describeIf,@@ -236,7 +237,7 @@ describe("Scrape tests", () => {);expect(response.markdown?.trim()).toContain(- process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],+ config.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],);},scrapeTimeout,@@ -255,7 +256,7 @@ describe("Scrape tests", () => {);expect(response.markdown?.trim()).toContain(- process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],+ config.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],);},scrapeTimeout,diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsnew file mode 100644index 000000000..1d8c28893--- /dev/null+++ b/apps/api/src/config.ts@@ -0,0 +1,192 @@+import "dotenv/config";+import { z } from "zod";++/* Codecs */+const delimitedList = (separator = ",") => {+ return z.codec(z.string(), z.array(z.string()), {+ decode: str => (str ? str.split(separator).map(s => s.trim()) : []),+ encode: arr => arr.join(separator),+ });+};++/* Schema */+const configSchema = z.object({+ // Application+ ENV: z.string().optional(),+ HOST: z.string().default("localhost"),+ PORT: z.coerce.number().default(3002),+ IS_PRODUCTION: z.stringbool().optional(),+ IS_KUBERNETES: z.stringbool().optional(),+ FIRECRAWL_APP_HOST: z.string().default("firecrawl-app-service"),+ FIRECRAWL_APP_PORT: z.string().default("3002"),+ FIRECRAWL_APP_SCHEME: z.string().default("http"),+ LOGGING_LEVEL: z.string().optional(),++ // Express+ EXPRESS_TRUST_PROXY: z.coerce.number().optional(),++ // API Keys & Authentication+ BULL_AUTH_KEY: z.string().optional(),+ OPENAI_API_KEY: z.string().optional(),+ OPENAI_BASE_URL: z.string().optional(),+ OPENROUTER_API_KEY: z.string().optional(),+ LLAMAPARSE_API_KEY: z.string().optional(),+ STRIPE_SECRET_KEY: z.string().optional(),+ RESEND_API_KEY: z.string().optional(),+ PREVIEW_TOKEN: z.string().optional(),+ SEARCH_PREVIEW_TOKEN: z.string().optional(),+ SEARCH_SERVICE_API_SECRET: z.string().optional(),++ // Database & Storage+ POSTGRES_HOST: z.string().default("localhost"),+ POSTGRES_PORT: z.string().default("5432"),+ POSTGRES_DB: z.string().default("postgres"),+ POSTGRES_USER: z.string().default("postgres"),+ POSTGRES_PASSWORD: z.string().default("postgres"),+ REDIS_URL: z.string().optional(),+ REDIS_EVICT_URL: z.string().optional(),+ REDIS_RATE_LIMIT_URL: z.string().optional(),+ NUQ_DATABASE_URL: z.string().optional(),+ NUQ_DATABASE_URL_LISTEN: z.string().optional(),+ NUQ_RABBITMQ_URL: z.string().optional(),++ // Supabase+ SUPABASE_URL: z.string().optional(),+ SUPABASE_ANON_TOKEN: z.string().optional(),+ SUPABASE_SERVICE_TOKEN: z.string().optional(),+ SUPABASE_REPLICA_URL: z.string().optional(),+ SUPABASE_ACUC_URL: z.string().optional(),+ SUPABASE_ACUC_SERVICE_TOKEN: z.string().optional(),+ INDEX_SUPABASE_URL: z.string().optional(),+ INDEX_SUPABASE_SERVICE_TOKEN: z.string().optional(),+ SEARCH_INDEX_SUPABASE_URL: z.string().optional(),++ // Google Cloud Storage+ GCS_BUCKET_NAME: z.string().optional(),+ GCS_CREDENTIALS: z.string().optional(),+ GCS_FIRE_ENGINE_BUCKET_NAME: z.string().optional(),+ GCS_INDEX_BUCKET_NAME: z.string().optional(),+ GCS_MEDIA_BUCKET_NAME: z.string().optional(),++ // Fire Engine+ FIRE_ENGINE_BETA_URL: z.string().optional(),+ FIRE_ENGINE_STAGING_URL: z.string().optional(),+ FIRE_ENGINE_AB_HOST: z.string().optional(),+ FIRE_ENGINE_AB_RATE: z.coerce.number().optional(),++ // ScrapeURL+ SCRAPEURL_AB_HOST: z.string().optional(),+ SCRAPEURL_AB_RATE: z.coerce.number().optional(),+ SCRAPEURL_AB_EXTEND_MAXAGE: z.stringbool().optional(),+ SCRAPEURL_ENGINE_WATERFALL_DELAY_MS: z.coerce.number().default(0),++ // Search Services+ SEARXNG_ENDPOINT: z.string().optional(),+ SEARXNG_ENGINES: z.string().optional(),+ SEARXNG_CATEGORIES: z.string().optional(),+ SEARCH_SERVICE_URL: z.string().optional(),+ SEARCH_INDEX_SAMPLE_RATE: z.coerce.number().default(0.1),+ ENABLE_SEARCH_INDEX: z.stringbool().optional(),++ // Worker Configuration+ WORKER_PORT: z.coerce.number().default(3005),+ NUQ_WORKER_PORT: z.coerce.number().default(3000),+ NUQ_WORKER_START_PORT: z.coerce.number().default(3006),+ NUQ_WORKER_COUNT: z.coerce.number().default(5),+ NUQ_PREFETCH_WORKER_PORT: z.coerce.number().default(3011),+ EXTRACT_WORKER_PORT: z.coerce.number().default(3004),+ NUQ_WAIT_MODE: z.string().optional(),++ // Job & Lock Management+ JOB_LOCK_EXTEND_INTERVAL: z.coerce.number().default(10000),+ JOB_LOCK_EXTENSION_TIME: z.coerce.number().default(60000),+ WORKER_LOCK_DURATION: z.coerce.number().default(60000),+ WORKER_STALLED_CHECK_INTERVAL: z.coerce.number().default(30000),+ CONNECTION_MONITOR_INTERVAL: z.coerce.number().default(10),+ CANT_ACCEPT_CONNECTION_INTERVAL: z.coerce.number().default(2000),++ // Proxy+ PROXY_SERVER: z.string().optional(),+ PROXY_USERNAME: z.string().optional(),+ PROXY_PASSWORD: z.string().optional(),++ // External Services+ PLAYWRIGHT_MICROSERVICE_URL: z.string().optional(),+ HTML_TO_MARKDOWN_SERVICE_URL: z.string().default("http://localhost:8080"),+ SMART_SCRAPE_API_URL: z.string().optional(),++ // PDF Processing+ PDF_MU_V2_BASE_URL: z.string().optional(),+ PDF_MU_V2_API_KEY: z.string().optional(),+ PDF_MU_V2_EXPERIMENT: z.string().optional(),+ PDF_MU_V2_EXPERIMENT_PERCENT: z.coerce.number().default(100),++ // RunPod+ RUNPOD_MU_API_KEY: z.string().optional(),+ RUNPOD_MU_POD_ID: z.string().optional(),++ // Webhooks+ SELF_HOSTED_WEBHOOK_URL: z.string().optional(),+ SELF_HOSTED_WEBHOOK_HMAC_SECRET: z.string().optional(),+ SLACK_WEBHOOK_URL: z.string().optional(),+ SLACK_ADMIN_WEBHOOK_URL: z.string().optional(),+ DISABLE_WEBHOOK_DELIVERY: z.stringbool().optional(),+ ALLOW_LOCAL_WEBHOOKS: z.stringbool().optional(),++ // Firecrawl Features+ FIRECRAWL_DEBUG_FILTER_LINKS: z.stringbool().optional(),+ FIRECRAWL_LOG_TO_FILE: z.stringbool().optional(),+ FIRECRAWL_SAVE_MOCKS: z.stringbool().optional(),+ FIRECRAWL_INDEX_WRITE_ONLY: z.stringbool().optional(),+ DISABLE_BLOCKLIST: z.stringbool().optional(),+ FORCED_ENGINE_DOMAINS: z.string().optional(),+ DEBUG_BRANDING: z.stringbool().optional(),++ // AI/ML+ MODEL_NAME: z.string().default("gpt-4o-mini"),+ MODEL_EMBEDDING_NAME: z.string().optional(),+ OLLAMA_BASE_URL: z.string().optional(),+ VERTEX_CREDENTIALS: z.string().optional(),++ // Rate Limiting+ RATE_LIMIT_TEST_API_KEY_SCRAPE: z.coerce.number().optional(),+ RATE_LIMIT_TEST_API_KEY_CRAWL: z.coerce.number().optional(),++ // Testing+ TEST_API_KEY: z.string().optional(),+ TEST_API_URL: z.string().default("http://127.0.0.1:3002"),+ TEST_TEAM_ID: z.string().optional(),+ TEST_SUITE_SELF_HOSTED: z.stringbool().optional(),+ TEST_SUITE_WEBSITE: z.string().default("http://127.0.0.1:4321"),+ USE_DB_AUTHENTICATION: z.stringbool().optional(),++ // Indexing+ BACKGROUND_INDEX_TEAM_ID: z.string().optional(),+ PRECRAWL_TEAM_ID: z.string().optional(),++ // Payment (x402)+ X402_ENDPOINT_PRICE_USD: z.string().optional(),+ X402_NETWORK: z.string().optional(),+ X402_PAY_TO_ADDRESS: z.string().optional(),++ // System+ MAX_CPU: z.coerce.number().default(0.8),+ MAX_RAM: z.coerce.number().default(0.8),+ SYS_INFO_MAX_CACHE_DURATION: z.coerce.number().default(150),+ USE_GO_MARKDOWN_PARSER: z.stringbool().optional(),++ // Sentry+ SENTRY_DSN: z.string().optional(),+ SENTRY_TRACE_SAMPLE_RATE: z.coerce.number().default(0.01),+ SENTRY_ERROR_SAMPLE_RATE: z.coerce.number().default(0.05),+ SENTRY_ENVIRONMENT: z.string().default("production"),+ NUQ_POD_NAME: z.string().default("main"),++ // Miscellaneous+ IDMUX_URL: z.string().optional(),+ GITHUB_RUN_NUMBER: z.string().optional(),+ GITHUB_REF_NAME: z.string().optional(),+ RESTRICTED_COUNTRIES: delimitedList(",").optional(),+});++export const config = configSchema.parse(process.env);diff --git a/apps/api/src/controllers/__tests__/crawl.test.ts b/apps/api/src/controllers/__tests__/crawl.test.tsindex 8d6b539d8..f2b0966ad 100644--- a/apps/api/src/controllers/__tests__/crawl.test.ts+++ b/apps/api/src/controllers/__tests__/crawl.test.ts@@ -1,4 +1,5 @@import { crawlController } from "../v0/crawl";+import { config } from "../../config";import { Request, Response } from "express";import { authenticateUser } from "../auth"; // Ensure this import is correctimport { createIdempotencyKey } from "../../services/idempotency/create";@@ -21,7 +22,7 @@ describe("crawlController", () => {const req = {headers: {"x-idempotency-key": await uuidv7(),- Authorization: `Bearer ${process.env.TEST_API_KEY}`,+ Authorization: `Bearer ${config.TEST_API_KEY}`,},body: {url: "https://mendable.ai",diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 3fd5379db..b67125418 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -1,4 +1,5 @@import { parseApi } from "../lib/parseApi";+import { config } from "../config";import { getRateLimiter } from "../services/rate-limiter";import { AuthResponse, NotificationType, RateLimiterMode } from "../types";import {@@ -151,16 +152,13 @@ export async function getACUC(mode === RateLimiterMode.ExtractStatus ||mode === RateLimiterMode.ExtractAgentPreview;- if (api_key === process.env.PREVIEW_TOKEN) {+ if (api_key === config.PREVIEW_TOKEN) {const acuc = mockPreviewACUC(api_key, isExtract);acuc.is_extract = isExtract;return acuc;}- if (- process.env.USE_DB_AUTHENTICATION !== "true" &&- !process.env.SUPABASE_ACUC_URL- ) {+ if (config.USE_DB_AUTHENTICATION !== true && !config.SUPABASE_ACUC_URL) {const acuc = mockACUC();acuc.is_extract = isExtract;return acuc;@@ -181,7 +179,7 @@ export async function getACUC(let retries = 0;const maxRetries = 5;while (retries < maxRetries) {- const client = !!process.env.SUPABASE_ACUC_URL+ const client = !!config.SUPABASE_ACUC_URL? supabase_acuc_only_service: Math.random() > 2 / 3? supabase_rr_service@@ -289,10 +287,7 @@ export async function getACUCTeam(return acuc;}- if (- process.env.USE_DB_AUTHENTICATION !== "true" &&- !process.env.SUPABASE_ACUC_URL- ) {+ if (config.USE_DB_AUTHENTICATION !== true && !config.SUPABASE_ACUC_URL) {const acuc = mockACUC();acuc.is_extract = isExtract;return acuc;@@ -314,7 +309,7 @@ export async function getACUCTeam(const maxRetries = 5;while (retries < maxRetries) {- const client = !!process.env.SUPABASE_ACUC_URL+ const client = !!config.SUPABASE_ACUC_URL? supabase_acuc_only_service: Math.random() > 2 / 3? supabase_rr_service@@ -396,7 +391,7 @@ export async function authenticateUser(res,mode?: RateLimiterMode,): Promise<AuthResponse> {- if (!!process.env.SUPABASE_ACUC_URL) {+ if (!!config.SUPABASE_ACUC_URL) {return supaAuthenticateUser(req, res, mode);}@@ -446,7 +441,7 @@ async function supaAuthenticateUser("Unauthenticated Playground calls are temporarily disabled due to abuse. Please sign up.",);}- if (token == process.env.PREVIEW_TOKEN) {+ if (token == config.PREVIEW_TOKEN) {if (mode == RateLimiterMode.CrawlStatus) {rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token);} else if (mode == RateLimiterMode.ExtractStatus) {@@ -487,8 +482,7 @@ async function supaAuthenticateUser();}- const team_endpoint_token =- token === process.env.PREVIEW_TOKEN ? iptoken : teamId;+ const team_endpoint_token = token === config.PREVIEW_TOKEN ? iptoken : teamId;try {await rateLimiter.consume(team_endpoint_token);@@ -518,7 +512,7 @@ async function supaAuthenticateUser(}if (- token === process.env.PREVIEW_TOKEN &&+ token === config.PREVIEW_TOKEN &&(mode === RateLimiterMode.Scrape ||mode === RateLimiterMode.Preview ||mode === RateLimiterMode.Map ||@@ -537,7 +531,7 @@ async function supaAuthenticateUser(// if (origin && origin.includes("firecrawl.dev")){// return { success: true, team_id: "preview" };// }- // if(process.env.ENV !== "production") {+ // if(config.ENV !== "production") {// return { success: true, team_id: "preview" };// }diff --git a/apps/api/src/controllers/v0/admin/check-fire-engine.ts b/apps/api/src/controllers/v0/admin/check-fire-engine.tsindex f8bf8beb8..6f90e8850 100644--- a/apps/api/src/controllers/v0/admin/check-fire-engine.ts+++ b/apps/api/src/controllers/v0/admin/check-fire-engine.ts@@ -1,10 +1,11 @@import { logger } from "../../../lib/logger";+import { config } from "../../../config";import * as Sentry from "@sentry/node";import { Request, Response } from "express";export async function checkFireEngine(req: Request, res: Response) {try {- if (!process.env.FIRE_ENGINE_BETA_URL) {+ if (!config.FIRE_ENGINE_BETA_URL) {logger.warn("Fire engine beta URL not configured");return res.status(500).json({success: false,@@ -20,21 +21,18 @@ export async function checkFireEngine(req: Request, res: Response) {for (const url of urls) {try {- const response = await fetch(- `${process.env.FIRE_ENGINE_BETA_URL}/scrape`,- {- method: "POST",- headers: {- "Content-Type": "application/json",- "X-Disable-Cache": "true",- },- body: JSON.stringify({- url,- engine: "chrome-cdp",- }),- signal: controller.signal,+ const response = await fetch(`${config.FIRE_ENGINE_BETA_URL}/scrape`, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "X-Disable-Cache": "true",},- );+ body: JSON.stringify({+ url,+ engine: "chrome-cdp",+ }),+ signal: controller.signal,+ });clearTimeout(timeout);diff --git a/apps/api/src/controllers/v0/admin/redis-health.ts b/apps/api/src/controllers/v0/admin/redis-health.tsindex c6f0419d2..53a281b82 100644--- a/apps/api/src/controllers/v0/admin/redis-health.ts+++ b/apps/api/src/controllers/v0/admin/redis-health.ts@@ -1,4 +1,5 @@import { Request, Response } from "express";+import { config } from "../../../config";import Redis from "ioredis";import { logger } from "../../../lib/logger";import { redisRateLimitClient } from "../../../services/rate-limiter";@@ -17,7 +18,7 @@ export async function redisHealthController(req: Request, res: Response) {};try {- const queueRedis = new Redis(process.env.REDIS_URL!);+ const queueRedis = new Redis(config.REDIS_URL!);const testKey = "test";const testValue = "test";diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex aea5775d3..7262c4d58 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -1,4 +1,5 @@import { Request, Response } from "express";+import { config } from "../../config";import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../../src/types";import { redisEvictConnection } from "../../../src/services/redis";@@ -20,10 +21,10 @@ async function getJobs(): Promise<PseudoJob<any>[]> {const [nuqJobs, dbScrapes, gcsJobs] = await Promise.all([scrapeQueue.getJobs(ids),- process.env.USE_DB_AUTHENTICATION === "true"+ config.USE_DB_AUTHENTICATION? await supabaseGetScrapesByRequestId(crawlId): [],- process.env.GCS_BUCKET_NAME+ config.GCS_BUCKET_NAME? (Promise.all(ids.map(async x => ({ id: x, job: await getJobFromGCS(x) })),).then(x => x.filter(x => x.job)) as Promise<diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex c6936e068..435250be4 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import { v7 as uuidv7 } from "uuid";import {BatchScrapeRequest,@@ -139,7 +140,7 @@ export async function batchScrapeController(...internalOptions,disableSmartWaitCache: true,teamId: req.auth.team_id,- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME? true: false,zeroDataRetention,diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex 55e1ad837..ffe321330 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import {CrawlErrorsResponse,CrawlStatusParams,@@ -72,7 +73,7 @@ export async function crawlErrorsController("crawl:" + req.params.jobId + ":robots_blocked",),});- } else if (process.env.USE_DB_AUTHENTICATION === "true") {+ } else if (config.USE_DB_AUTHENTICATION) {// Check the requests table for the crawl/batch scrape requestconst { data: request, error: requestError } = await supabase_rr_service.from("requests")diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 7bb1c7b46..f39d4103a 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import {CrawlStatusParams,CrawlStatusResponse,@@ -54,12 +55,10 @@ export type DBScrape = {export async function getJob(id: string): Promise<PseudoJob<any> | null> {const [nuqJob, dbScrape, gcsJob] = await Promise.all([scrapeQueue.getJob(id) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,- (process.env.USE_DB_AUTHENTICATION === "true"+ (config.USE_DB_AUTHENTICATION? supabaseGetScrapeById(id): null) as Promise<DBScrape | null>,- (process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<- any | null- >,+ (config.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);if (!nuqJob && !dbScrape) return null;@@ -98,10 +97,8 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const [nuqJobs, dbScrapes, gcsJobs] = await Promise.all([scrapeQueue.getJobs(ids) as Promise<NuQJob<ScrapeJobSingleUrls>[]>,- process.env.USE_DB_AUTHENTICATION === "true"- ? supabaseGetScrapesById(ids)- : [],- process.env.GCS_BUCKET_NAME+ config.USE_DB_AUTHENTICATION ? supabaseGetScrapesById(ids) : [],+ config.GCS_BUCKET_NAME? (Promise.all(ids.map(async x => ({ id: x, job: await getJobFromGCS(x) })),).then(x => x.filter(x => x.job)) as Promise<@@ -198,16 +195,15 @@ export async function crawlStatusController(logger.child({ zeroDataRetention }),);- const creditsRpc =- process.env.USE_DB_AUTHENTICATION === "true"- ? await supabase_service.rpc(- "credits_billed_by_crawl_id_2",- {- i_crawl_id: req.params.jobId,- },- { get: true },- )- : null;+ const creditsRpc = config.USE_DB_AUTHENTICATION+ ? await supabase_service.rpc(+ "credits_billed_by_crawl_id_2",+ {+ i_crawl_id: req.params.jobId,+ },+ { get: true },+ )+ : null;let outputBulkA: {status?: "completed" | "scraping" | "cancelled";diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex b67415cd3..bd57db1bd 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import { v7 as uuidv7 } from "uuid";import {CrawlRequest,@@ -65,7 +66,7 @@ export async function crawlController(});let { remainingCredits } = req.account!;- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;if (!useDbAuthentication) {remainingCredits = Infinity;}@@ -121,9 +122,7 @@ export async function crawlController(...internalOptions,disableSmartWaitCache: true,teamId: req.auth.team_id,- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME- ? true- : false,+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,zeroDataRetention,}, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.tsindex abfa06775..907819331 100644--- a/apps/api/src/controllers/v1/extract-status.ts+++ b/apps/api/src/controllers/v1/extract-status.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import { RequestWithAuth } from "./types";import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";import { getExtractQueue } from "../../services/queue-service";@@ -33,7 +34,7 @@ async function getExtractJob(): Promise<ExtractPseudoJob<ExtractResult> | null> {const [bullJob, dbExtract] = await Promise.all([getExtractQueue().getJob(id),- (process.env.USE_DB_AUTHENTICATION === "true"+ (config.USE_DB_AUTHENTICATION? supabaseGetExtractByIdDirect(id): null) as Promise<DBExtract | null>,]);diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex cd3eb1218..eb2d08e29 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -24,6 +24,7 @@ import {import { createWebhookSender, WebhookEvent } from "../../services/webhook";import { logRequest } from "../../services/logging/log_job";+import { config } from "../../config";async function oldExtract(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,res: Response<ExtractResponse>,@@ -113,7 +114,7 @@ export async function extractController(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,res: Response<ExtractResponse>,) {- const selfHosted = process.env.USE_DB_AUTHENTICATION !== "true";+ const selfHosted = config.USE_DB_AUTHENTICATION !== true;const originalRequest = { ...req.body };req.body = extractRequestSchema.parse(req.body);diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex ca514e954..772ae393d 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import { v7 as uuidv7 } from "uuid";import {MapDocument,@@ -31,7 +32,7 @@ import { MapTimeoutError } from "../../lib/error";import { checkPermissions } from "../../lib/permissions";configDotenv();-const redis = new Redis(process.env.REDIS_URL!);+const redis = new Redis(config.REDIS_URL!);// Max Links that "Smart /map" can returnconst MAX_FIRE_ENGINE_RESULTS = 500;diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 47e850017..d7af937f6 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import { logger as _logger } from "../../lib/logger";import {Document,@@ -81,8 +82,8 @@ export async function scrapeController(// const startTime = new Date().getTime();const isDirectToBullMQ =- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;+ config.SEARCH_PREVIEW_TOKEN !== undefined &&+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(req.body,@@ -147,7 +148,7 @@ export async function scrapeController(internalOptions: {...internalOptions,teamId: req.auth.team_id,- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME? true: false,unnormalizedSourceURL: preNormalizedBody.url,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex f901a13fa..0fcacdc10 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import {Document,RequestWithAuth,@@ -170,7 +171,7 @@ async function scrapeSearchResult(};let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;- if (process.env.USE_DB_AUTHENTICATION === "true") {+ if (config.USE_DB_AUTHENTICATION) {const { data: costTrackingResponse, error: costTrackingError } =await supabase_service.from("scrapes")@@ -256,8 +257,8 @@ export async function searchController(};const middlewareTime = controllerStartTime - middlewareStartTime;const isSearchPreview =- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;+ config.SEARCH_PREVIEW_TOKEN !== undefined &&+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;let credits_billed = 0;let allDocsWithCostTracking: DocumentWithCostTracking[] = [];diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 1768632d3..ad360009f 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1,4 +1,5 @@import { Request, Response } from "express";+import { config } from "../../config";import { z } from "zod";import { protocolIncluded, checkUrl } from "../../lib/validateUrl";import { countries } from "../../lib/validate-country";@@ -52,10 +53,7 @@ export const url = z.preprocess(.url().regex(/^https?:\/\//i, "URL uses unsupported protocol").refine(x => {- if (- process.env.TEST_SUITE_SELF_HOSTED === "true" &&- process.env.ALLOW_LOCAL_WEBHOOKS === "true"- ) {+ if (config.TEST_SUITE_SELF_HOSTED && config.ALLOW_LOCAL_WEBHOOKS) {if (/^https?:\/\/(localhost|127\.0\.0\.1|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})(:\d+)?([\/?#]|$)/i.test(x as string,diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex 7cfb39530..ff5f9b0e2 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import {Document,RequestWithAuth,@@ -120,7 +121,7 @@ async function scrapeX402SearchResult(};let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;- if (process.env.USE_DB_AUTHENTICATION === "true") {+ if (config.USE_DB_AUTHENTICATION) {const { data: costTrackingResponse, error: costTrackingError } =await supabase_service.from("scrapes")@@ -197,8 +198,8 @@ export async function x402SearchController(};const startTime = new Date().getTime();const isSearchPreview =- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;+ config.SEARCH_PREVIEW_TOKEN !== undefined &&+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;try {req.body = searchRequestSchema.parse(req.body);diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex e530df0c3..30c503959 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import { v7 as uuidv7 } from "uuid";import {BatchScrapeRequest,@@ -132,7 +133,7 @@ export async function batchScrapeController(internalOptions: {disableSmartWaitCache: true,teamId: req.auth.team_id,- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME? true: false,zeroDataRetention,diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsindex 55e1ad837..ffe321330 100644--- a/apps/api/src/controllers/v2/crawl-errors.ts+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import {CrawlErrorsResponse,CrawlStatusParams,@@ -72,7 +73,7 @@ export async function crawlErrorsController("crawl:" + req.params.jobId + ":robots_blocked",),});- } else if (process.env.USE_DB_AUTHENTICATION === "true") {+ } else if (config.USE_DB_AUTHENTICATION) {// Check the requests table for the crawl/batch scrape requestconst { data: request, error: requestError } = await supabase_rr_service.from("requests")diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 077fb565f..9d29491e0 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import {CrawlStatusParams,CrawlStatusResponse,@@ -62,12 +63,10 @@ export async function getJob(id,_logger,) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,- (process.env.USE_DB_AUTHENTICATION === "true"+ (config.USE_DB_AUTHENTICATION? supabaseGetScrapeById(id): null) as Promise<DBScrape | null>,- (process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<- any | null- >,+ (config.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);if (!nuqJob && !dbScrape) return null;@@ -109,10 +108,8 @@ export async function getJobs(): Promise<PseudoJob<any>[]> {const [nuqJobs, dbScrapes, gcsJobs] = await Promise.all([scrapeQueue.getJobs(ids, _logger) as Promise<NuQJob<ScrapeJobSingleUrls>[]>,- process.env.USE_DB_AUTHENTICATION === "true"- ? supabaseGetScrapesById(ids)- : [],- process.env.GCS_BUCKET_NAME+ config.USE_DB_AUTHENTICATION ? supabaseGetScrapesById(ids) : [],+ config.GCS_BUCKET_NAME? (Promise.all(ids.map(async x => ({ id: x, job: await getJobFromGCS(x) })),).then(x => x.filter(x => x.job)) as Promise<@@ -218,16 +215,15 @@ export async function crawlStatusController(logger.child({ zeroDataRetention }),);- const creditsRpc =- process.env.USE_DB_AUTHENTICATION === "true"- ? await supabase_service.rpc(- "credits_billed_by_crawl_id_2",- {- i_crawl_id: req.params.jobId,- },- { get: true },- )- : null;+ const creditsRpc = config.USE_DB_AUTHENTICATION+ ? await supabase_service.rpc(+ "credits_billed_by_crawl_id_2",+ {+ i_crawl_id: req.params.jobId,+ },+ { get: true },+ )+ : null;let outputBulkA: {status?: "completed" | "scraping" | "cancelled";diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex babd3a0dd..4534e01fa 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import { v7 as uuidv7 } from "uuid";import {CrawlRequest,@@ -72,7 +73,7 @@ export async function crawlController(});let { remainingCredits } = req.account!;- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;if (!useDbAuthentication) {remainingCredits = Infinity;}@@ -185,9 +186,7 @@ export async function crawlController(internalOptions: {disableSmartWaitCache: true,teamId: req.auth.team_id,- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME- ? true- : false,+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,zeroDataRetention,},team_id: req.auth.team_id,diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.tsindex 3c55c14e6..378554c0a 100644--- a/apps/api/src/controllers/v2/extract-status.ts+++ b/apps/api/src/controllers/v2/extract-status.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import { RequestWithAuth } from "./types";import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";import { getExtractQueue } from "../../services/queue-service";@@ -33,7 +34,7 @@ async function getExtractJob(): Promise<ExtractPseudoJob<ExtractResult> | null> {const [bullJob, dbExtract] = await Promise.all([getExtractQueue().getJob(id),- (process.env.USE_DB_AUTHENTICATION === "true"+ (config.USE_DB_AUTHENTICATION? supabaseGetExtractByIdDirect(id): null) as Promise<DBExtract | null>,]);diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 78c005902..6fd95c75e 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import { logger as _logger } from "../../lib/logger";import {Document,@@ -120,8 +121,8 @@ export async function scrapeController(const timeout = req.body.timeout;const isDirectToBullMQ =- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;+ config.SEARCH_PREVIEW_TOKEN !== undefined &&+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;const totalWait =(req.body.waitFor ?? 0) +diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 98f171084..a85924c51 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import {Document,RequestWithAuth,@@ -161,7 +162,7 @@ async function scrapeSearchResult(};let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;- if (process.env.USE_DB_AUTHENTICATION === "true") {+ if (config.USE_DB_AUTHENTICATION) {const { data: costTrackingResponse, error: costTrackingError } =await supabase_service.from("scrapes")@@ -236,8 +237,8 @@ export async function searchController(const middlewareTime = controllerStartTime - middlewareStartTime;const isSearchPreview =- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;+ config.SEARCH_PREVIEW_TOKEN !== undefined &&+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;let credits_billed = 0;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex c3ff9869c..c5de2d7c5 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1,4 +1,5 @@import { Request, Response } from "express";+import { config } from "../../config";import { z } from "zod";import { protocolIncluded, checkUrl } from "../../lib/validateUrl";import { countries } from "../../lib/validate-country";@@ -46,10 +47,7 @@ const BASE_URL_SCHEMA = z.preprocess(.url().regex(/^https?:\/\//i, "URL uses unsupported protocol").refine(x => {- if (- process.env.TEST_SUITE_SELF_HOSTED === "true" &&- process.env.ALLOW_LOCAL_WEBHOOKS === "true"- ) {+ if (config.TEST_SUITE_SELF_HOSTED && config.ALLOW_LOCAL_WEBHOOKS) {if (/^https?:\/\/(localhost|127\.0\.0\.1|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})(:\d+)?([\/?#]|$)/i.test(x as string,diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex 81e6fdfd0..bc09fb7cb 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { config } from "../../config";import {Document,RequestWithAuth,@@ -148,7 +149,7 @@ async function scrapeX420SearchResult(};let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;- if (process.env.USE_DB_AUTHENTICATION === "true") {+ if (config.USE_DB_AUTHENTICATION) {const { data: costTrackingResponse, error: costTrackingError } =await supabase_service.from("scrapes")@@ -218,8 +219,8 @@ export async function x402SearchController(const startTime = new Date().getTime();const isSearchPreview =- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;+ config.SEARCH_PREVIEW_TOKEN !== undefined &&+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;let credits_billed = 0;diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex 56a8b4c9a..a3b43697b 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -1,4 +1,4 @@-import "dotenv/config";+import { config } from "./config";import { type ChildProcess, spawn } from "child_process";import * as net from "net";import { basename, join } from "path";@@ -87,21 +87,19 @@ function formatDuration(nanoseconds: bigint): string {return `${minutes}m ${remainingSeconds.toFixed(0)}s`;}-const PORT = process.env.PORT ?? "3002";-const WORKER_PORT = process.env.WORKER_PORT ?? "3005";-const EXTRACT_WORKER_PORT = process.env.EXTRACT_WORKER_PORT ?? "3004";-const NUQ_WORKER_START_PORT = Number(- process.env.NUQ_WORKER_START_PORT ?? "3006",-);-const NUQ_WORKER_COUNT = Number(process.env.NUQ_WORKER_COUNT ?? "5");+const PORT = config.PORT;+const WORKER_PORT = config.WORKER_PORT;+const EXTRACT_WORKER_PORT = config.EXTRACT_WORKER_PORT;+const NUQ_WORKER_START_PORT = config.NUQ_WORKER_START_PORT;+const NUQ_WORKER_COUNT = config.NUQ_WORKER_COUNT;const NUQ_PREFETCH_WORKER_PORT = NUQ_WORKER_START_PORT + NUQ_WORKER_COUNT;// PostgreSQL credentials (with defaults for backward compatibility)-const POSTGRES_USER = process.env.POSTGRES_USER ?? "postgres";-const POSTGRES_PASSWORD = process.env.POSTGRES_PASSWORD ?? "postgres";-const POSTGRES_DB = process.env.POSTGRES_DB ?? "postgres";-const POSTGRES_HOST = process.env.POSTGRES_HOST ?? "localhost";-const POSTGRES_PORT = process.env.POSTGRES_PORT ?? "5432";+const POSTGRES_USER = config.POSTGRES_USER;+const POSTGRES_PASSWORD = config.POSTGRES_PASSWORD;+const POSTGRES_DB = config.POSTGRES_DB;+const POSTGRES_HOST = config.POSTGRES_HOST;+const POSTGRES_PORT = config.POSTGRES_PORT;// Shell escape helper to prevent command injectionfunction shellEscape(arg: string): string {@@ -492,7 +490,7 @@ async function waitForPostgres(async function setupNuqPostgres(): Promise<Services["nuqPostgres"]> {// If NUQ_DATABASE_URL is already set, respect it (user's explicit choice)- if (process.env.NUQ_DATABASE_URL) {+ if (config.NUQ_DATABASE_URL) {logger.info("NUQ_DATABASE_URL is set, skipping container management");return undefined;}@@ -504,6 +502,8 @@ async function setupNuqPostgres(): Promise<Services["nuqPostgres"]> {// Running in docker-compose: construct URL with proper encodinglogger.section("Setting up NUQ PostgreSQL connection for docker-compose");const dbUrl = `postgresql://${encodeURIComponent(POSTGRES_USER)}:${encodeURIComponent(POSTGRES_PASSWORD)}@${POSTGRES_HOST}:${POSTGRES_PORT}/${encodeURIComponent(POSTGRES_DB)}`;+ config.NUQ_DATABASE_URL = dbUrl;+ config.NUQ_DATABASE_URL_LISTEN = dbUrl;process.env.NUQ_DATABASE_URL = dbUrl;process.env.NUQ_DATABASE_URL_LISTEN = dbUrl;logger.success(@@ -540,6 +540,8 @@ async function setupNuqPostgres(): Promise<Services["nuqPostgres"]> {// Set environment variables for the services with proper encodingconst dbUrl = `postgresql://${encodeURIComponent(POSTGRES_USER)}:${encodeURIComponent(POSTGRES_PASSWORD)}@localhost:5432/${encodeURIComponent(POSTGRES_DB)}`;+ config.NUQ_DATABASE_URL = dbUrl;+ config.NUQ_DATABASE_URL_LISTEN = dbUrl;process.env.NUQ_DATABASE_URL = dbUrl;process.env.NUQ_DATABASE_URL_LISTEN = dbUrl;@@ -623,7 +625,7 @@ async function startServices(command?: string[]): Promise<Services> {{NUQ_REDUCE_NOISE: "true",NUQ_POD_NAME: "worker",- WORKER_PORT: WORKER_PORT,+ WORKER_PORT: String(WORKER_PORT),},);@@ -635,7 +637,7 @@ async function startServices(command?: string[]): Promise<Services> {{NUQ_REDUCE_NOISE: "true",NUQ_POD_NAME: "extract-worker",- EXTRACT_WORKER_PORT: EXTRACT_WORKER_PORT,+ EXTRACT_WORKER_PORT: String(EXTRACT_WORKER_PORT),},);@@ -653,7 +655,7 @@ async function startServices(command?: string[]): Promise<Services> {),);- const nuqPrefetchWorker = process.env.NUQ_RABBITMQ_URL+ const nuqPrefetchWorker = config.NUQ_RABBITMQ_URL? execForward("nuq-prefetch-worker",process.argv[2] === "--start-docker"@@ -668,19 +670,18 @@ async function startServices(command?: string[]): Promise<Services> {): undefined;- const indexWorker =- process.env.USE_DB_AUTHENTICATION === "true"- ? execForward(- "index-worker",- process.argv[2] === "--start-docker"- ? "node dist/src/services/indexing/index-worker.js"- : "pnpm index-worker:production",- {- NUQ_REDUCE_NOISE: "true",- NUQ_POD_NAME: "index-worker",- },- )- : undefined;+ const indexWorker = config.USE_DB_AUTHENTICATION+ ? execForward(+ "index-worker",+ process.argv[2] === "--start-docker"+ ? "node dist/src/services/indexing/index-worker.js"+ : "pnpm index-worker:production",+ {+ NUQ_REDUCE_NOISE: "true",+ NUQ_POD_NAME: "index-worker",+ },+ )+ : undefined;// tests hammer the API instantly, so we need to ensure it's running before launching testsif (diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 1b227273c..4e9a6b5b2 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -1,4 +1,5 @@import "dotenv/config";+import { config } from "./config";import "./services/sentry";import { setSentryServiceTag } from "./services/sentry";import * as Sentry from "@sentry/node";@@ -41,7 +42,7 @@ const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");const { ExpressAdapter } = require("@bull-board/express");-const numCPUs = process.env.ENV === "local" ? 2 : os.cpus().length;+const numCPUs = config.ENV === "local" ? 2 : os.cpus().length;logger.info(`Number of CPUs: ${numCPUs} available`);logger.info("Network info dump", {@@ -57,7 +58,7 @@ const expressApp = express();const ws = expressWs(expressApp);const app = ws.app;-global.isProduction = process.env.IS_PRODUCTION === "true";+global.isProduction = config.IS_PRODUCTION;setSentryServiceTag("api");@@ -70,12 +71,12 @@ app.use(responseTime());app.disable("x-powered-by");-if (process.env.EXPRESS_TRUST_PROXY) {- app.set("trust proxy", parseInt(process.env.EXPRESS_TRUST_PROXY, 10));+if (config.EXPRESS_TRUST_PROXY) {+ app.set("trust proxy", config.EXPRESS_TRUST_PROXY);}const serverAdapter = new ExpressAdapter();-serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);+serverAdapter.setBasePath(`/admin/${config.BULL_AUTH_KEY}/queues`);const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({queues: [@@ -88,10 +89,7 @@ const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({serverAdapter: serverAdapter,});-app.use(- `/admin/${process.env.BULL_AUTH_KEY}/queues`,- serverAdapter.getRouter(),-);+app.use(`/admin/${config.BULL_AUTH_KEY}/queues`, serverAdapter.getRouter());app.get("/", (_, res) => {res.redirect("https://docs.firecrawl.dev/api-reference/v2-introduction");@@ -108,8 +106,8 @@ app.use("/v2", v2Router);app.use(adminRouter);app.use(domainFrequencyRouter);-const DEFAULT_PORT = process.env.PORT ?? 3002;-const HOST = process.env.HOST ?? "localhost";+const DEFAULT_PORT = config.PORT;+const HOST = config.HOST;async function startServer(port = DEFAULT_PORT) {try {@@ -131,7 +129,7 @@ async function startServer(port = DEFAULT_PORT) {const exitHandler = async () => {logger.info("SIGTERM signal received: closing HTTP server");- if (process.env.IS_KUBERNETES === "true") {+ if (config.IS_KUBERNETES) {// Account for GCE load balancer drain timeoutlogger.info("Waiting 60s for GCE load balancer drain timeout");await new Promise(resolve => setTimeout(resolve, 60000));diff --git a/apps/api/src/lib/branding/transformer.ts b/apps/api/src/lib/branding/transformer.tsindex 5e64dcf97..8a6471448 100644--- a/apps/api/src/lib/branding/transformer.ts+++ b/apps/api/src/lib/branding/transformer.ts@@ -1,4 +1,5 @@import { processRawBranding } from "./processor";+import { config } from "../../config";import { BrandingProfile } from "../../types/branding";import { enhanceBrandingWithLLM } from "./llm";import { Meta } from "../../scraper/scrapeURL";@@ -94,7 +95,7 @@ export async function brandingTransformer(brandingProfile = jsBranding;}- if (process.env.DEBUG_BRANDING !== "true") {+ if (config.DEBUG_BRANDING !== true) {delete (brandingProfile as any).__button_snapshots;}diff --git a/apps/api/src/lib/deployment.ts b/apps/api/src/lib/deployment.tsindex e83e4063e..0e3ccc539 100644--- a/apps/api/src/lib/deployment.ts+++ b/apps/api/src/lib/deployment.ts@@ -1,5 +1,6 @@+import { config } from "../config";export function isSelfHosted(): boolean {- return process.env.USE_DB_AUTHENTICATION !== "true";+ return config.USE_DB_AUTHENTICATION !== true;}export function getErrorContactMessage(errorId?: string): string {diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex d5d702143..c91765748 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -1,4 +1,5 @@import { v7 as uuidv7 } from "uuid";+import { config } from "../../config";import {Document,ScrapeOptions,@@ -59,7 +60,7 @@ export async function scrapeDocument(}),internalOptions: {teamId: options.teamId,- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME? true: false,bypassBilling: true,diff --git a/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts b/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.tsindex ab6db63f6..04d9e5c09 100644--- a/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts+++ b/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts@@ -1,4 +1,5 @@import { TokenUsage } from "../../../../controllers/v1/types";+import { config } from "../../../../config";import { logger } from "../../../../lib/logger";import { modelPrices } from "../../usage/model-prices";@@ -26,7 +27,7 @@ export function estimateTotalCost_F0(tokenUsage: TokenUsage[]): number {export function estimateCost_F0(tokenUsage: TokenUsage): number {let totalCost = 0;try {- let model = tokenUsage.model ?? (process.env.MODEL_NAME || "gpt-4o-mini");+ let model = tokenUsage.model ?? config.MODEL_NAME;const pricing = modelPrices[model] as ModelPricing;if (!pricing) {diff --git a/apps/api/src/lib/extract/usage/llm-cost.ts b/apps/api/src/lib/extract/usage/llm-cost.tsindex 4a9fd3a95..a24b4ff4c 100644--- a/apps/api/src/lib/extract/usage/llm-cost.ts+++ b/apps/api/src/lib/extract/usage/llm-cost.ts@@ -1,4 +1,5 @@import { TokenUsage } from "../../../controllers/v1/types";+import { config } from "../../../config";import { logger } from "../../../lib/logger";import { CostTracking } from "../../cost-tracking";import { modelPrices } from "./model-prices";@@ -31,7 +32,7 @@ export function estimateTotalCost(tokenUsage: TokenUsage[]): number {function estimateCost(tokenUsage: TokenUsage): number {let totalCost = 0;try {- let model = tokenUsage.model ?? (process.env.MODEL_NAME || "gpt-4o-mini");+ let model = tokenUsage.model ?? config.MODEL_NAME;const pricing = modelPrices[model] as ModelPricing;if (!pricing) {diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.tsindex 62bdd0280..6896c5914 100644--- a/apps/api/src/lib/gcs-jobs.ts+++ b/apps/api/src/lib/gcs-jobs.ts@@ -10,9 +10,10 @@ import type {LoggedScrape,LoggedSearch,} from "../services/logging/log_job";+import { config } from "../config";-const credentials = process.env.GCS_CREDENTIALS- ? JSON.parse(atob(process.env.GCS_CREDENTIALS))+const credentials = config.GCS_CREDENTIALS+ ? JSON.parse(atob(config.GCS_CREDENTIALS)): undefined;export const storage = new Storage({ credentials });@@ -27,12 +28,12 @@ export async function saveScrapeToGCS(scrape: LoggedScrape): Promise<void> {"job.num_docs": 1,});- if (!process.env.GCS_BUCKET_NAME) {+ if (!config.GCS_BUCKET_NAME) {setSpanAttributes(span, { "gcs.bucket_configured": false });return;}- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${scrape.id}.json`);// Save job docs with retry@@ -119,12 +120,12 @@ export async function saveSearchToGCS(search: LoggedSearch): Promise<void> {request_id: search.request_id,});- if (!process.env.GCS_BUCKET_NAME) {+ if (!config.GCS_BUCKET_NAME) {setSpanAttributes(span, { "gcs.bucket_configured": false });return;}- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${search.id}.json`);for (let i = 0; i < 3; i++) {@@ -197,12 +198,12 @@ export async function saveExtractToGCS(extract: LoggedExtract): Promise<void> {"extract.team_id": extract.team_id,});- if (!process.env.GCS_BUCKET_NAME) {+ if (!config.GCS_BUCKET_NAME) {setSpanAttributes(span, { "gcs.bucket_configured": false });return;}- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${extract.id}.json`);for (let i = 0; i < 3; i++) {@@ -271,12 +272,12 @@ export async function saveMapToGCS(map: LoggedMap): Promise<void> {"map.team_id": map.team_id,});- if (!process.env.GCS_BUCKET_NAME) {+ if (!config.GCS_BUCKET_NAME) {setSpanAttributes(span, { "gcs.bucket_configured": false });return;}- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${map.id}.json`);for (let i = 0; i < 3; i++) {@@ -345,12 +346,12 @@ export async function saveDeepResearchToGCS("deep_research.team_id": deepResearch.team_id,});- if (!process.env.GCS_BUCKET_NAME) {+ if (!config.GCS_BUCKET_NAME) {setSpanAttributes(span, { "gcs.bucket_configured": false });return;}- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${deepResearch.id}.json`);for (let i = 0; i < 3; i++) {@@ -418,12 +419,12 @@ export async function saveLlmsTxtToGCS(llmsTxt: LoggedLlmsTxt): Promise<void> {"llms_txt.team_id": llmsTxt.team_id,});- if (!process.env.GCS_BUCKET_NAME) {+ if (!config.GCS_BUCKET_NAME) {setSpanAttributes(span, { "gcs.bucket_configured": false });return;}- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${llmsTxt.id}.json`);for (let i = 0; i < 3; i++) {@@ -491,12 +492,12 @@ export async function getJobFromGCS(jobId: string): Promise<Document[] | null> {"job.id": jobId,});- if (!process.env.GCS_BUCKET_NAME) {+ if (!config.GCS_BUCKET_NAME) {setSpanAttributes(span, { "gcs.bucket_configured": false });return null;}- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${jobId}.json`);try {@@ -531,12 +532,12 @@ export async function removeJobFromGCS(jobId: string): Promise<void> {"job.id": jobId,});- if (!process.env.GCS_BUCKET_NAME) {+ if (!config.GCS_BUCKET_NAME) {setSpanAttributes(span, { "gcs.bucket_configured": false });return;}- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${jobId}.json`);try {@@ -570,11 +571,11 @@ export async function getDocFromGCS(url: string): Promise<any | null> {// url,// });try {- if (!process.env.GCS_FIRE_ENGINE_BUCKET_NAME) {+ if (!config.GCS_FIRE_ENGINE_BUCKET_NAME) {return null;}- const bucket = storage.bucket(process.env.GCS_FIRE_ENGINE_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_FIRE_ENGINE_BUCKET_NAME);const blob = bucket.file(`${url}`);const [exists] = await blob.exists();if (!exists) {diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex 5f42a5734..0e7273559 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -1,9 +1,10 @@import { logger } from "./logger";+import { config } from "../config";import crypto from "crypto";import { storage } from "./gcs-jobs";-const credentials = process.env.GCS_CREDENTIALS- ? JSON.parse(atob(process.env.GCS_CREDENTIALS))+const credentials = config.GCS_CREDENTIALS+ ? JSON.parse(atob(config.GCS_CREDENTIALS)): undefined;const PDF_CACHE_PREFIX = "pdf-cache-v2/";@@ -23,12 +24,12 @@ export async function savePdfResultToCache(result: { markdown: string; html: string },): Promise<string | null> {try {- if (!process.env.GCS_BUCKET_NAME) {+ if (!config.GCS_BUCKET_NAME) {return null;}const cacheKey = createPdfCacheKey(pdfContent);- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);for (let i = 0; i < 3; i++) {@@ -79,12 +80,12 @@ export async function getPdfResultFromCache(pdfContent: string,): Promise<{ markdown: string; html: string } | null> {try {- if (!process.env.GCS_BUCKET_NAME) {+ if (!config.GCS_BUCKET_NAME) {return null;}const cacheKey = createPdfCacheKey(pdfContent);- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);const [exists] = await blob.exists();diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.tsindex eb37f6dce..a7372e3fe 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts@@ -1,4 +1,5 @@import { supabase_service } from "../../services/supabase";+import { config } from "../../config";import { logger } from "../logger";import { normalizeUrl, normalizeUrlOnlyHostname } from "../canonical-url";@@ -13,7 +14,7 @@ export async function getLlmsTextFromCache(url: string,maxUrls: number,): Promise<LlmsTextCache | null> {- if (process.env.USE_DB_AUTHENTICATION !== "true") {+ if (config.USE_DB_AUTHENTICATION !== true) {return null;}@@ -54,7 +55,7 @@ export async function saveLlmsTextToCache(llmstxt_full: string,maxUrls: number,): Promise<void> {- if (process.env.USE_DB_AUTHENTICATION !== "true") {+ if (config.USE_DB_AUTHENTICATION !== true) {return;}diff --git a/apps/api/src/lib/generic-ai.ts b/apps/api/src/lib/generic-ai.tsindex 85192990f..cc86e5e46 100644--- a/apps/api/src/lib/generic-ai.ts+++ b/apps/api/src/lib/generic-ai.ts@@ -1,4 +1,5 @@import { createOpenAI } from "@ai-sdk/openai";+import { config } from "../config";import { createOllama } from "ollama-ai-provider";import { anthropic } from "@ai-sdk/anthropic";import { groq } from "@ai-sdk/groq";@@ -18,23 +19,21 @@ type Provider =| "fireworks"| "deepinfra"| "vertex";-const defaultProvider: Provider = process.env.OLLAMA_BASE_URL- ? "ollama"- : "openai";+const defaultProvider: Provider = config.OLLAMA_BASE_URL ? "ollama" : "openai";const providerList: Record<Provider, any> = {openai: createOpenAI({- apiKey: process.env.OPENAI_API_KEY,- baseURL: process.env.OPENAI_BASE_URL,+ apiKey: config.OPENAI_API_KEY,+ baseURL: config.OPENAI_BASE_URL,}), //OPENAI_API_KEYollama: createOllama({- baseURL: process.env.OLLAMA_BASE_URL,+ baseURL: config.OLLAMA_BASE_URL,}),anthropic, //ANTHROPIC_API_KEYgroq, //GROQ_API_KEYgoogle, //GOOGLE_GENERATIVE_AI_API_KEYopenrouter: createOpenRouter({- apiKey: process.env.OPENROUTER_API_KEY,+ apiKey: config.OPENROUTER_API_KEY,}),fireworks, //FIREWORKS_API_KEYdeepinfra, //DEEPINFRA_API_KEY@@ -44,9 +43,9 @@ const providerList: Record<Provider, any> = {baseURL:"https://aiplatform.googleapis.com/v1/projects/firecrawl/locations/global/publishers/google",location: "global",- googleAuthOptions: process.env.VERTEX_CREDENTIALS+ googleAuthOptions: config.VERTEX_CREDENTIALS? {- credentials: JSON.parse(atob(process.env.VERTEX_CREDENTIALS)),+ credentials: JSON.parse(atob(config.VERTEX_CREDENTIALS)),}: {keyFile: "./gke-key.json",@@ -58,8 +57,8 @@ export function getModel(name: string, provider: Provider = defaultProvider) {if (name === "gemini-2.5-pro") {name = "gemini-2.5-pro";}- return process.env.MODEL_NAME- ? providerList[provider](process.env.MODEL_NAME)+ return config.MODEL_NAME+ ? providerList[provider](config.MODEL_NAME): providerList[provider](name);}@@ -67,7 +66,7 @@ export function getEmbeddingModel(name: string,provider: Provider = defaultProvider,) {- return process.env.MODEL_EMBEDDING_NAME- ? providerList[provider].embedding(process.env.MODEL_EMBEDDING_NAME)+ return config.MODEL_EMBEDDING_NAME+ ? providerList[provider].embedding(config.MODEL_EMBEDDING_NAME): providerList[provider].embedding(name);}diff --git a/apps/api/src/lib/html-to-markdown-client.ts b/apps/api/src/lib/html-to-markdown-client.tsindex 0541f4234..aca0a676b 100644--- a/apps/api/src/lib/html-to-markdown-client.ts+++ b/apps/api/src/lib/html-to-markdown-client.ts@@ -6,6 +6,7 @@*/import axios, { AxiosInstance, AxiosError } from "axios";+import { config } from "../config";import { logger } from "./logger";import * as Sentry from "@sentry/node";@@ -38,10 +39,7 @@ export async function convertHTMLToMarkdownWithHttpService(return "";}- const url =- serviceUrl ||- process.env.HTML_TO_MARKDOWN_SERVICE_URL ||- "http://localhost:8080";+ const url = serviceUrl || config.HTML_TO_MARKDOWN_SERVICE_URL;const startTime = Date.now();try {diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.tsindex 04b162bda..b7d2f3848 100644--- a/apps/api/src/lib/html-to-markdown.ts+++ b/apps/api/src/lib/html-to-markdown.ts@@ -1,14 +1,12 @@import koffi from "koffi";+import { config } from "../config";import "../services/sentry";import * as Sentry from "@sentry/node";--import dotenv from "dotenv";import { logger } from "./logger";import { stat } from "fs/promises";import { HTML_TO_MARKDOWN_PATH } from "../natives";import { convertHTMLToMarkdownWithHttpService } from "./html-to-markdown-client";import { postProcessMarkdown } from "@mendable/firecrawl-rs";-dotenv.config();// TODO: add a timeout to the Go parser@@ -60,7 +58,7 @@ export async function parseMarkdown(}// Try HTTP service first if enabled- if (process.env.HTML_TO_MARKDOWN_SERVICE_URL) {+ if (config.HTML_TO_MARKDOWN_SERVICE_URL) {try {let markdownContent = await convertHTMLToMarkdownWithHttpService(html);markdownContent = await postProcessMarkdown(markdownContent);@@ -79,7 +77,7 @@ export async function parseMarkdown(}try {- if (process.env.USE_GO_MARKDOWN_PARSER == "true") {+ if (config.USE_GO_MARKDOWN_PARSER) {const converter = await GoMarkdownConverter.getInstance();let markdownContent = await converter.convertHTMLToMarkdown(html);markdownContent = await postProcessMarkdown(markdownContent);diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.tsindex 9c61cb9a2..ad6052c09 100644--- a/apps/api/src/lib/logger.ts+++ b/apps/api/src/lib/logger.ts@@ -1,5 +1,6 @@import * as winston from "winston";+import { config } from "../config";import { configDotenv } from "dotenv";configDotenv();@@ -36,7 +37,7 @@ const zeroDataRetentionFilter = winston.format(info => {})();export const logger = winston.createLogger({- level: process.env.LOGGING_LEVEL?.toLowerCase() ?? "debug",+ level: config.LOGGING_LEVEL?.toLowerCase() ?? "debug",format: winston.format.json({replacer(key, value) {if (value instanceof Error) {@@ -53,7 +54,7 @@ export const logger = winston.createLogger({},}),transports: [- ...(process.env.FIRECRAWL_LOG_TO_FILE+ ...(config.FIRECRAWL_LOG_TO_FILE? [new winston.transports.File({filename:@@ -77,9 +78,9 @@ export const logger = winston.createLogger({winston.format.metadata({fillExcept: ["message", "level", "timestamp"],}),- ...((process.env.ENV === "production" &&- process.env.SENTRY_ENVIRONMENT === "dev") ||- process.env.ENV !== "production"+ ...((config.ENV === "production" &&+ config.SENTRY_ENVIRONMENT === "dev") ||+ config.ENV !== "production"? [winston.format.colorize(), logFormat]: []),),diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex 6a53018d3..ab2194f24 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -1,4 +1,5 @@import robotsParser, { Robot } from "robots-parser";+import { config } from "../config";import { Logger } from "winston";import { ScrapeOptions, scrapeOptions } from "../controllers/v2/types";import { scrapeURL } from "../scraper/scrapeURL";@@ -6,8 +7,8 @@ import { Engine } from "../scraper/scrapeURL/engines";import { CostTracking } from "./cost-tracking";const useFireEngine =- process.env.FIRE_ENGINE_BETA_URL !== "" &&- process.env.FIRE_ENGINE_BETA_URL !== undefined;+ config.FIRE_ENGINE_BETA_URL !== "" &&+ config.FIRE_ENGINE_BETA_URL !== undefined;interface RobotsTxtChecker {robotsTxtUrl: string;diff --git a/apps/api/src/lib/search-index-client.ts b/apps/api/src/lib/search-index-client.tsindex dd0f1da67..6423accde 100644--- a/apps/api/src/lib/search-index-client.ts+++ b/apps/api/src/lib/search-index-client.ts@@ -6,6 +6,7 @@*/import { logger as _logger } from "./logger";+import { config } from "../config";import type { Logger } from "winston";interface SearchIndexClientConfig {@@ -86,10 +87,7 @@ export class SearchIndexClient {* Check if search index service is enabled*/static isEnabled(): boolean {- return !!(- process.env.SEARCH_SERVICE_URL &&- process.env.ENABLE_SEARCH_INDEX === "true"- );+ return !!(config.SEARCH_SERVICE_URL && config.ENABLE_SEARCH_INDEX);}/**@@ -321,8 +319,8 @@ export function getSearchIndexClient(): SearchIndexClient | null {}if (!searchIndexClient) {- const baseUrl = process.env.SEARCH_SERVICE_URL!;- const apiSecret = process.env.SEARCH_SERVICE_API_SECRET;+ const baseUrl = config.SEARCH_SERVICE_URL!;+ const apiSecret = config.SEARCH_SERVICE_API_SECRET;searchIndexClient = new SearchIndexClient({baseUrl,diff --git a/apps/api/src/lib/withAuth.ts b/apps/api/src/lib/withAuth.tsindex bec3d4d1a..a9d602cbd 100644--- a/apps/api/src/lib/withAuth.ts+++ b/apps/api/src/lib/withAuth.ts@@ -2,6 +2,7 @@ import { AuthResponse } from "../../src/types";import { logger } from "./logger";import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";+import { config } from "../config";configDotenv();let warningCount = 0;@@ -11,7 +12,7 @@ export function withAuth<T, U extends any[]>(mockSuccess: T,) {return async function (...args: U): Promise<T> {- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;if (!useDbAuthentication) {if (warningCount < 5) {logger.warn("You're bypassing authentication");diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex 723d35ec3..cc6839595 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -1,4 +1,5 @@import express from "express";+import { config } from "../config";import { redisHealthController } from "../controllers/v0/admin/redis-health";import { wrap } from "./shared";import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";@@ -19,57 +20,54 @@ import { integValidateApiKeyController } from "../controllers/v0/admin/validate-export const adminRouter = express.Router();adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/redis-health`,+ `/admin/${config.BULL_AUTH_KEY}/redis-health`,redisHealthController,);adminRouter.post(- `/admin/${process.env.BULL_AUTH_KEY}/acuc-cache-clear`,+ `/admin/${config.BULL_AUTH_KEY}/acuc-cache-clear`,wrap(acucCacheClearController),);adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/feng-check`,+ `/admin/${config.BULL_AUTH_KEY}/feng-check`,wrap(checkFireEngine),);-adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/cclog`,- wrap(cclogController),-);+adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/cclog`, wrap(cclogController));adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/zdrcleaner`,+ `/admin/${config.BULL_AUTH_KEY}/zdrcleaner`,wrap(zdrcleanerController),);adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/index-queue-prometheus`,+ `/admin/${config.BULL_AUTH_KEY}/index-queue-prometheus`,wrap(indexQueuePrometheus),);adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/precrawl`,+ `/admin/${config.BULL_AUTH_KEY}/precrawl`,wrap(triggerPrecrawl),);adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/metrics`,+ `/admin/${config.BULL_AUTH_KEY}/metrics`,wrap(metricsController),);adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/nuq-metrics`,+ `/admin/${config.BULL_AUTH_KEY}/nuq-metrics`,wrap(nuqMetricsController),);adminRouter.post(- `/admin/${process.env.BULL_AUTH_KEY}/fsearch`,+ `/admin/${config.BULL_AUTH_KEY}/fsearch`,wrap(realtimeSearchController),);adminRouter.post(- `/admin/${process.env.BULL_AUTH_KEY}/concurrency-queue-backfill`,+ `/admin/${config.BULL_AUTH_KEY}/concurrency-queue-backfill`,wrap(concurrencyQueueBackfillController),);diff --git a/apps/api/src/routes/domain-frequency.ts b/apps/api/src/routes/domain-frequency.tsindex e25fd7bce..2d7391c2a 100644--- a/apps/api/src/routes/domain-frequency.ts+++ b/apps/api/src/routes/domain-frequency.ts@@ -1,4 +1,5 @@import express from "express";+import { config } from "../config";import { Request, Response } from "express";import {getTopDomains,@@ -12,7 +13,7 @@ const domainFrequencyRouter = express.Router();// Get top domains by frequencydomainFrequencyRouter.get(- `/domain-frequency/${process.env.BULL_AUTH_KEY}/top`,+ `/domain-frequency/${config.BULL_AUTH_KEY}/top`,async (req: Request, res: Response) => {try {const limit = parseInt(req.query.limit as string) || 100;@@ -34,7 +35,7 @@ domainFrequencyRouter.get(// Get frequency for a specific domaindomainFrequencyRouter.get(- `/domain-frequency/${process.env.BULL_AUTH_KEY}/domain/:domain`,+ `/domain-frequency/${config.BULL_AUTH_KEY}/domain/:domain`,async (req: Request, res: Response) => {try {const { domain } = req.params;@@ -70,7 +71,7 @@ domainFrequencyRouter.get(// Get overall statisticsdomainFrequencyRouter.get(- `/domain-frequency/${process.env.BULL_AUTH_KEY}/stats`,+ `/domain-frequency/${config.BULL_AUTH_KEY}/stats`,async (req: Request, res: Response) => {try {const [stats, queueLength] = await Promise.all([diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex faa07fe1a..8e7dba30c 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -19,6 +19,7 @@ import * as geoip from "geoip-country";import { isSelfHosted } from "../lib/deployment";import { validate as isUuid } from "uuid";+import { config } from "../config";export function checkCreditsMiddleware(_minimum?: number,): (req: RequestWithAuth, res: Response, next: NextFunction) => void {@@ -230,8 +231,7 @@ export function countryCheck(return next();}- const restricted = process.env.RESTRICTED_COUNTRIES?.split(",") ?? [];- if (restricted.includes(country.country)) {+ if (config.RESTRICTED_COUNTRIES?.includes(country.country)) {logger.warn("Denied access to restricted country", {ip: req.ip,country: country.country,diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex f4a20f896..c187ee3d1 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -1,4 +1,5 @@import express from "express";+import { config } from "../config";import { crawlController } from "../controllers/v1/crawl";// import { crawlStatusController } from "../../src/controllers/v1/crawl-status";import { scrapeController } from "../../src/controllers/v1/scrape";@@ -51,12 +52,12 @@ v1Router.use(requestTimingMiddleware("v1"));// x402 payments protocol - https://github.com/coinbase/x402// v1Router.use(// paymentMiddleware(-// (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||+// (config.X402_PAY_TO_ADDRESS as `0x${string}`) ||// "0x0000000000000000000000000000000000000000",// {// "POST /x402/search": {-// price: process.env.X402_ENDPOINT_PRICE_USD as string,-// network: process.env.X402_NETWORK as+// price: config.X402_ENDPOINT_PRICE_USD as string,+// network: config.X402_NETWORK as// | "base-sepolia"// | "base"// | "avalanche-fuji"@@ -311,12 +312,12 @@ v1Router.post(authMiddleware(RateLimiterMode.Search),countryCheck,paymentMiddleware(- (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||+ (config.X402_PAY_TO_ADDRESS as `0x${string}`) ||"0x0000000000000000000000000000000000000000",{"POST /x402/search": {- price: process.env.X402_ENDPOINT_PRICE_USD as string,- network: process.env.X402_NETWORK as+ price: config.X402_ENDPOINT_PRICE_USD as string,+ network: config.X402_NETWORK as| "base-sepolia"| "base"| "avalanche-fuji"diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex f088fb632..68b39f45b 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -1,4 +1,5 @@import express from "express";+import { config } from "../config";import { RateLimiterMode } from "../types";import expressWs from "express-ws";import { searchController } from "../controllers/v2/search";@@ -48,12 +49,12 @@ v2Router.use(requestTimingMiddleware("v2"));// x402 payments protocol - https://github.com/coinbase/x402// v2Router.use(// paymentMiddleware(-// (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||+// (config.X402_PAY_TO_ADDRESS as `0x${string}`) ||// "0x0000000000000000000000000000000000000000",// {// "POST /x402/search": {-// price: process.env.X402_ENDPOINT_PRICE_USD as string,-// network: process.env.X402_NETWORK as+// price: config.X402_ENDPOINT_PRICE_USD as string,+// network: config.X402_NETWORK as// | "base-sepolia"// | "base"// | "avalanche-fuji"@@ -340,12 +341,12 @@ v2Router.post(countryCheck,blocklistMiddleware,paymentMiddleware(- (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||+ (config.X402_PAY_TO_ADDRESS as `0x${string}`) ||"0x0000000000000000000000000000000000000000",{"POST /x402/search": {- price: process.env.X402_ENDPOINT_PRICE_USD as string,- network: process.env.X402_NETWORK as+ price: config.X402_ENDPOINT_PRICE_USD as string,+ network: config.X402_NETWORK as| "base-sepolia"| "base"| "avalanche-fuji"diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex e6e66b42a..410fe79d6 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -1,4 +1,5 @@import { AxiosError } from "axios";+import { config } from "../../config";import { load } from "cheerio"; // rustifiedimport { URL } from "url";import { getLinksFromSitemap } from "./sitemap";@@ -186,7 +187,7 @@ export class WebCrawler {fancyDenialReasons.set(key, DenialReason[value]);});- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {for (const link of res.links) {this.logger.debug(`${link} OK`);}@@ -233,7 +234,7 @@ export class WebCrawler {"file:",];if (nonWebProtocols.some(protocol => urlStr.startsWith(protocol))) {- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} NON-WEB PROTOCOL FAIL`);}denialReasons.set(link, DenialReason.NON_WEB_PROTOCOL);@@ -244,7 +245,7 @@ export class WebCrawler {// Check if the link exceeds the maximum depth allowedif (depth > maxDepth) {- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} DEPTH FAIL`);}denialReasons.set(link, DenialReason.DEPTH_LIMIT);@@ -260,7 +261,7 @@ export class WebCrawler {new RegExp(excludePattern).test(excincPath),)) {- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} EXCLUDE FAIL`);}denialReasons.set(link, DenialReason.EXCLUDE_PATTERN);@@ -275,7 +276,7 @@ export class WebCrawler {new RegExp(includePattern).test(excincPath),)) {- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} INCLUDE FAIL`);}denialReasons.set(link, DenialReason.INCLUDE_PATTERN);@@ -289,7 +290,7 @@ export class WebCrawler {try {normalizedLink = new URL(link);} catch (_) {- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} URL PARSE FAIL`);}return false;@@ -310,7 +311,7 @@ export class WebCrawler {if (!normalizedLink.pathname.startsWith(normalizedInitialUrl.pathname)) {- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} BACKWARDS FAIL ${normalizedLink.pathname} ${normalizedInitialUrl.pathname}`,);@@ -332,7 +333,7 @@ export class WebCrawler {method: "filterLinks",link,});- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} ROBOTS FAIL`);}denialReasons.set(link, DenialReason.ROBOTS_TXT);@@ -340,14 +341,14 @@ export class WebCrawler {}if (this.isFile(link)) {- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} FILE FAIL`);}denialReasons.set(link, DenialReason.FILE_TYPE);return false;}- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {this.logger.debug(`${link} OK`);}return true;diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 7c28a867e..d7faa57bd 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -1,4 +1,5 @@import { parseStringPromise } from "xml2js";+import { config } from "../../config";import { WebCrawler, SITEMAP_LIMIT } from "./crawler";import { scrapeURL } from "../scrapeURL";import { scrapeOptions } from "../../controllers/v2/types";@@ -19,8 +20,8 @@ import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";import { useIndex } from "../../services";const useFireEngine =- process.env.FIRE_ENGINE_BETA_URL !== "" &&- process.env.FIRE_ENGINE_BETA_URL !== undefined;+ config.FIRE_ENGINE_BETA_URL !== "" &&+ config.FIRE_ENGINE_BETA_URL !== undefined;const gunzipAsync = promisify(gunzip);diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/engine-forcing.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/engine-forcing.test.tsindex a0972f73f..f1f4469cd 100644--- a/apps/api/src/scraper/WebScraper/utils/__tests__/engine-forcing.test.ts+++ b/apps/api/src/scraper/WebScraper/utils/__tests__/engine-forcing.test.ts@@ -1,8 +1,8 @@import { initializeEngineForcing, getEngineForUrl } from "../engine-forcing";-+import { config } from "../../../../config";describe("Engine Forcing", () => {beforeEach(() => {- delete process.env.FORCED_ENGINE_DOMAINS;+ delete config.FORCED_ENGINE_DOMAINS;});describe("initializeEngineForcing", () => {@@ -12,13 +12,13 @@ describe("Engine Forcing", () => {});it("should initialize with empty mappings when env var is empty", () => {- process.env.FORCED_ENGINE_DOMAINS = "";+ config.FORCED_ENGINE_DOMAINS = "";initializeEngineForcing();expect(getEngineForUrl("https://example.com")).toBeUndefined();});it("should parse valid JSON mappings", () => {- process.env.FORCED_ENGINE_DOMAINS = JSON.stringify({+ config.FORCED_ENGINE_DOMAINS = JSON.stringify({"example.com": "playwright",});initializeEngineForcing();@@ -26,7 +26,7 @@ describe("Engine Forcing", () => {});it("should handle invalid JSON gracefully", () => {- process.env.FORCED_ENGINE_DOMAINS = "invalid json";+ config.FORCED_ENGINE_DOMAINS = "invalid json";initializeEngineForcing();expect(getEngineForUrl("https://example.com")).toBeUndefined();});@@ -34,7 +34,7 @@ describe("Engine Forcing", () => {describe("getEngineForUrl", () => {beforeEach(() => {- process.env.FORCED_ENGINE_DOMAINS = JSON.stringify({+ config.FORCED_ENGINE_DOMAINS = JSON.stringify({"example.com": "playwright","test.com": "fetch","*.subdomain.com": "fire-engine;chrome-cdp",@@ -93,7 +93,7 @@ describe("Engine Forcing", () => {});it("should throw error if not initialized", () => {- delete process.env.FORCED_ENGINE_DOMAINS;+ delete config.FORCED_ENGINE_DOMAINS;expect(() => {const {getEngineForUrl: freshGetEngineForUrl,diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex 1f7a3966d..f375f384c 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -1,4 +1,5 @@import { configDotenv } from "dotenv";+import { config } from "../../../config";import { parse } from "tldts";import { TeamFlags } from "../../../controllers/v1/types";import { supabase_rr_service } from "../../../services/supabase";@@ -13,10 +14,7 @@ type BlocklistBlob = {let blob: BlocklistBlob | null = null;export async function initializeBlocklist() {- if (- process.env.USE_DB_AUTHENTICATION !== "true" ||- process.env.DISABLE_BLOCKLIST === "true"- ) {+ if (config.USE_DB_AUTHENTICATION !== true || config.DISABLE_BLOCKLIST) {blob = {blocklist: [],allowedKeywords: [],diff --git a/apps/api/src/scraper/WebScraper/utils/engine-forcing.ts b/apps/api/src/scraper/WebScraper/utils/engine-forcing.tsindex 89c972978..4aff39f66 100644--- a/apps/api/src/scraper/WebScraper/utils/engine-forcing.ts+++ b/apps/api/src/scraper/WebScraper/utils/engine-forcing.ts@@ -3,6 +3,7 @@ import { parse } from "tldts";import { Engine } from "../../scrapeURL/engines";import { logger } from "../../../lib/logger";+import { config } from "../../../config";configDotenv();type EngineForcingMapping = {@@ -17,7 +18,7 @@ let engineMappings: EngineForcingMapping | null = null;* Example: {"example.com": "playwright", "*.google.com": ["fire-engine;chrome-cdp", "playwright"]}*/export function initializeEngineForcing() {- const envVar = process.env.FORCED_ENGINE_DOMAINS;+ const envVar = config.FORCED_ENGINE_DOMAINS;if (!envVar || envVar.trim() === "") {engineMappings = {};diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.tsindex c24bdc4ba..7319d9ce9 100644--- a/apps/api/src/scraper/crawler/sitemap.ts+++ b/apps/api/src/scraper/crawler/sitemap.ts@@ -1,4 +1,5 @@import type { Logger } from "winston";+import { config } from "../../config";import { scrapeOptions, ScrapeOptions } from "../../controllers/v2/types";import { logger as _logger } from "../../lib/logger";import { Engine } from "../scrapeURL/engines";@@ -15,8 +16,8 @@ import { SitemapError } from "../../lib/error";import { useIndex } from "../../services";const useFireEngine =- process.env.FIRE_ENGINE_BETA_URL !== "" &&- process.env.FIRE_ENGINE_BETA_URL !== undefined;+ config.FIRE_ENGINE_BETA_URL !== "" &&+ config.FIRE_ENGINE_BETA_URL !== undefined;type SitemapScrapeOptions = {url: string;diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 7b5fd03b0..1d91851e8 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -19,6 +19,7 @@ import {import { Meta } from "../..";import { abTestFireEngine } from "../../../../services/ab-test";+import { config } from "../../../../config";export type FireEngineScrapeRequestCommon = {url: string;@@ -168,9 +169,9 @@ const failedSchema = z.object({});export const fireEngineURL =- process.env.FIRE_ENGINE_BETA_URL ?? "<mock-fire-engine-url>";+ config.FIRE_ENGINE_BETA_URL ?? "<mock-fire-engine-url>";export const fireEngineStagingURL =- process.env.FIRE_ENGINE_STAGING_URL ?? "<mock-fire-engine-url>";+ config.FIRE_ENGINE_STAGING_URL ?? "<mock-fire-engine-url>";export async function fireEngineScrape<Engine extendsdiff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 4e29065b2..0e54bdcec 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -1,4 +1,5 @@import { ScrapeActionContent } from "../../../lib/entities";+import { config } from "../../../config";import { Meta } from "..";import { documentMaxReasonableTime, scrapeDocument } from "./document";import {@@ -37,11 +38,11 @@ export type Engine =| "index;documents";const useFireEngine =- process.env.FIRE_ENGINE_BETA_URL !== "" &&- process.env.FIRE_ENGINE_BETA_URL !== undefined;+ config.FIRE_ENGINE_BETA_URL !== "" &&+ config.FIRE_ENGINE_BETA_URL !== undefined;const usePlaywright =- process.env.PLAYWRIGHT_MICROSERVICE_URL !== "" &&- process.env.PLAYWRIGHT_MICROSERVICE_URL !== undefined;+ config.PLAYWRIGHT_MICROSERVICE_URL !== "" &&+ config.PLAYWRIGHT_MICROSERVICE_URL !== undefined;const engines: Engine[] = [...(useIndex ? ["index" as const, "index;documents" as const] : []),@@ -465,7 +466,7 @@ const engineOptions: {export function shouldUseIndex(meta: Meta) {return (useIndex &&- process.env.FIRECRAWL_INDEX_WRITE_ONLY !== "true" &&+ config.FIRECRAWL_INDEX_WRITE_ONLY !== true &&!hasFormatOfType(meta.options.formats, "changeTracking") &&!hasFormatOfType(meta.options.formats, "branding") &&// Skip index if a non-default PDF maxPages is specifieddiff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 7906fc9ac..06425d702 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -1,4 +1,5 @@import crypto from "crypto";+import { config } from "../../../../config";import { Document } from "../../../../controllers/v1/types";import { EngineScrapeResult } from "..";import { Meta } from "../..";@@ -208,8 +209,8 @@ export async function scrapeURLWithIndex(if (domainSplitsHash.length === 0 ||- process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true" ||- process.env.USE_DB_AUTHENTICATION !== "true"+ config.FIRECRAWL_INDEX_WRITE_ONLY ||+ config.USE_DB_AUTHENTICATION !== true) {maxAge = 2 * 24 * 60 * 60 * 1000; // 2 days} else {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 07e5f4146..b8420e8aa 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -1,4 +1,5 @@import { Meta } from "../..";+import { config } from "../../../../config";import { EngineScrapeResult } from "..";import * as marked from "marked";import { robustFetch } from "../../lib/fetch";@@ -68,10 +69,9 @@ async function scrapePDFWithRunPodMU(});if (- process.env.PDF_MU_V2_EXPERIMENT === "true" &&- process.env.PDF_MU_V2_BASE_URL &&- Math.random() * 100 <- Number(process.env.PDF_MU_V2_EXPERIMENT_PERCENT ?? "100")+ config.PDF_MU_V2_EXPERIMENT === "true" &&+ config.PDF_MU_V2_BASE_URL &&+ Math.random() * 100 < config.PDF_MU_V2_EXPERIMENT_PERCENT) {(async () => {const pdfParseId = crypto.randomUUID();@@ -85,9 +85,11 @@ async function scrapePDFWithRunPodMU(});try {const resp = await robustFetch({- url: process.env.PDF_MU_V2_BASE_URL ?? "",+ url: config.PDF_MU_V2_BASE_URL ?? "",method: "POST",- headers: process.env.PDF_MU_V2_API_KEY ? { Authorization: `Bearer ${process.env.PDF_MU_V2_API_KEY}` } : undefined,+ headers: config.PDF_MU_V2_API_KEY+ ? { Authorization: `Bearer ${config.PDF_MU_V2_API_KEY}` }+ : undefined,body: {input: {file_content: base64Content,@@ -124,11 +126,10 @@ async function scrapePDFWithRunPodMU(const muV1StartedAt = Date.now();const podStart = await robustFetch({- url:- "https://api.runpod.ai/v2/" + process.env.RUNPOD_MU_POD_ID + "/runsync",+ url: "https://api.runpod.ai/v2/" + config.RUNPOD_MU_POD_ID + "/runsync",method: "POST",headers: {- Authorization: `Bearer ${process.env.RUNPOD_MU_API_KEY}`,+ Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,},body: {input: {@@ -164,10 +165,10 @@ async function scrapePDFWithRunPodMU(await new Promise(resolve => setTimeout(resolve, 2500));meta.abort.throwIfAborted();const podStatus = await robustFetch({- url: `https://api.runpod.ai/v2/${process.env.RUNPOD_MU_POD_ID}/status/${podStart.id}`,+ url: `https://api.runpod.ai/v2/${config.RUNPOD_MU_POD_ID}/status/${podStart.id}`,method: "GET",headers: {- Authorization: `Bearer ${process.env.RUNPOD_MU_API_KEY}`,+ Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,},logger: meta.logger.child({method: "scrapePDFWithRunPodMU/status/robustFetch",@@ -359,8 +360,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {// First try RunPod MU if conditions are metif (base64Content.length < MAX_FILE_SIZE &&- process.env.RUNPOD_MU_API_KEY &&- process.env.RUNPOD_MU_POD_ID+ config.RUNPOD_MU_API_KEY &&+ config.RUNPOD_MU_POD_ID) {const muV1StartedAt = Date.now();try {diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.tsindex ae2bfec97..1acb185b9 100644--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts@@ -1,4 +1,5 @@import { z } from "zod";+import { config } from "../../../../config";import { EngineScrapeResult } from "..";import { Meta } from "../..";import { robustFetch } from "../../lib/fetch";@@ -8,7 +9,7 @@ export async function scrapeURLWithPlaywright(meta: Meta,): Promise<EngineScrapeResult> {const response = await robustFetch({- url: process.env.PLAYWRIGHT_MICROSERVICE_URL!,+ url: config.PLAYWRIGHT_MICROSERVICE_URL!,headers: {"Content-Type": "application/json",},diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex a430e50bb..597560258 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -1,10 +1,10 @@import type { Socket } from "net";+import { config } from "../../../../config";import type { TLSSocket } from "tls";import * as undici from "undici";import { CookieJar } from "tough-cookie";import { cookie } from "http-cookie-agent/undici";import IPAddr from "ipaddr.js";-export class InsecureConnectionError extends Error {constructor() {super("Connection violated security rules.");@@ -23,13 +23,13 @@ function createBaseAgent(skipTlsVerification: boolean) {maxRedirections: 5000,};- return process.env.PROXY_SERVER+ return config.PROXY_SERVER? new undici.ProxyAgent({- uri: process.env.PROXY_SERVER.includes("://")- ? process.env.PROXY_SERVER- : "http://" + process.env.PROXY_SERVER,- token: process.env.PROXY_USERNAME- ? `Basic ${Buffer.from(process.env.PROXY_USERNAME + ":" + (process.env.PROXY_PASSWORD ?? "")).toString("base64")}`+ uri: config.PROXY_SERVER.includes("://")+ ? config.PROXY_SERVER+ : "http://" + config.PROXY_SERVER,+ token: config.PROXY_USERNAME+ ? `Basic ${Buffer.from(config.PROXY_USERNAME + ":" + (config.PROXY_PASSWORD ?? "")).toString("base64")}`: undefined,requestTls: {rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested@@ -55,7 +55,7 @@ function attachSecurityCheck(agent: undici.Dispatcher) {if (socket.remoteAddress &&isIPPrivate(socket.remoteAddress) &&- process.env.ALLOW_LOCAL_WEBHOOKS !== "true"+ config.ALLOW_LOCAL_WEBHOOKS !== true) {socket.destroy(new InsecureConnectionError());}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 0126b2e4b..9305a2c23 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -1,4 +1,5 @@import { Logger } from "winston";+import { config } from "../../config";import * as Sentry from "@sentry/node";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";@@ -499,7 +500,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {const waitUntilWaterfall =getEngineMaxReasonableTime(meta, engine) +- parseInt(process.env.SCRAPEURL_ENGINE_WATERFALL_DELAY_MS || "0", 10);+ config.SCRAPEURL_ENGINE_WATERFALL_DELAY_MS;if (!isFinite(waitUntilWaterfall) ||@@ -923,7 +924,7 @@ export async function scrapeURL(useIndex &&meta.options.storeInCache &&!meta.internalOptions.zeroDataRetention &&- internalOptions.teamId !== process.env.PRECRAWL_TEAM_ID &&+ internalOptions.teamId !== config.PRECRAWL_TEAM_ID &&meta.internalOptions.isPreCrawl !== true; // sitemap crawls override teamId but keep the isPreCrawl flagif (shouldRecordFrequency) {(async () => {diff --git a/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts b/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.tsindex dd7c6a0a5..a08076875 100644--- a/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts+++ b/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts@@ -1,7 +1,8 @@import CacheableLookup from "cacheable-lookup";import dns from "dns";+import { config } from "../../../config";export const cacheableLookup =- process.env.SENTRY_ENVIRONMENT === "dev"+ config.SENTRY_ENVIRONMENT === "dev"? { lookup: dns.lookup, install: () => {} }: new CacheableLookup({});diff --git a/apps/api/src/scraper/scrapeURL/lib/mock.ts b/apps/api/src/scraper/scrapeURL/lib/mock.tsindex d01dc690e..168b9b35d 100644--- a/apps/api/src/scraper/scrapeURL/lib/mock.ts+++ b/apps/api/src/scraper/scrapeURL/lib/mock.ts@@ -1,4 +1,5 @@import * as fs from "fs/promises";+import { config } from "../../../config";import * as path from "path";import { logger as _logger } from "../../../lib/logger";import { Logger } from "winston";@@ -8,7 +9,7 @@ const loadMocksDirPath = path.replace("dist/", "");export async function saveMock(options: unknown, result: unknown) {- if (process.env.FIRECRAWL_SAVE_MOCKS !== "true") return;+ if (config.FIRECRAWL_SAVE_MOCKS !== true) return;await fs.mkdir(saveMocksDirPath, { recursive: true });diff --git a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/smartScrape.tsindex 93decdaf2..8c9304881 100644--- a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts@@ -1,4 +1,5 @@import { z } from "zod";+import { config } from "../../../config";import { logger as _logger } from "../../../lib/logger";import { robustFetch } from "./fetch";import fs from "fs/promises";@@ -79,7 +80,7 @@ export async function smartScrape({// Pass schema type as generic parameter to robustFethconst response = await robustFetch<typeof smartScrapeResultSchema>({- url: `${process.env.SMART_SCRAPE_API_URL}/smart-scrape`,+ url: `${config.SMART_SCRAPE_API_URL}/smart-scrape`,method: "POST",body: {url,diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.tsindex e8336d999..f183df28e 100644--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts@@ -1,6 +1,7 @@import "dotenv/config";-process.env.ENV = "test";+import { config } from "../../config";+config.ENV = "test";import { scrapeURL } from ".";import { scrapeOptions } from "../../controllers/v2/types";diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.tsindex d630a5f28..01dc391b0 100644--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts@@ -10,6 +10,7 @@*/import { Document } from "../../../controllers/v1/types";+import { config } from "../../../config";import { indexDocumentIfEnabled } from "../../../lib/search-index-client";import { logger as _logger } from "../../../lib/logger";import { Meta } from "..";@@ -55,13 +56,12 @@ function shouldIndexForSearch(meta: Meta, document: Document): boolean {*/function shouldSampleDocument(): boolean {// Get sample rate from environment (default 10% for safe rollout)- const sampleRateStr = process.env.SEARCH_INDEX_SAMPLE_RATE || "0.1";- const sampleRate = parseFloat(sampleRateStr);+ const sampleRate = config.SEARCH_INDEX_SAMPLE_RATE;// Validate sample rateif (isNaN(sampleRate) || sampleRate < 0 || sampleRate > 1) {_logger.warn("Invalid SEARCH_INDEX_SAMPLE_RATE, using 0.1 (10%)", {- value: sampleRateStr,+ value: sampleRate,});return Math.random() < 0.1;}@@ -79,8 +79,7 @@ export async function sendDocumentToSearchIndex(): Promise<Document> {// Check if search indexing is enabled via the SEARCH_SERVICE_URLconst searchIndexEnabled =- process.env.ENABLE_SEARCH_INDEX === "true" &&- process.env.SEARCH_SERVICE_URL;+ config.ENABLE_SEARCH_INDEX && config.SEARCH_SERVICE_URL;meta.logger.debug("Sending document to search index", {url: meta.url,@@ -95,7 +94,7 @@ export async function sendDocumentToSearchIndex(if (!shouldSampleDocument()) {meta.logger.debug("Document not sampled for search indexing", {url: meta.url,- sampleRate: process.env.SEARCH_INDEX_SAMPLE_RATE || "0.1",+ sampleRate: config.SEARCH_INDEX_SAMPLE_RATE,});return document;}diff --git a/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts b/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.tsindex 83df17b84..55cb5c9f8 100644--- a/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts@@ -2,12 +2,13 @@// and it makes me sad. - mogeryimport { supabase_service } from "../../../services/supabase";+import { config } from "../../../config";import { Meta } from "..";import { Document } from "../../../controllers/v1/types";export function uploadScreenshot(meta: Meta, document: Document): Document {if (- process.env.USE_DB_AUTHENTICATION === "true" &&+ config.USE_DB_AUTHENTICATION &&document.screenshot !== undefined &&document.screenshot.startsWith("data:")) {diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.tsindex f7264756d..8d3f55cf4 100644--- a/apps/api/src/search/fireEngine.ts+++ b/apps/api/src/search/fireEngine.ts@@ -1,14 +1,12 @@-import dotenv from "dotenv";+import { config } from "../config";import { SearchResult } from "../../src/lib/entities";import * as Sentry from "@sentry/node";import { logger } from "../lib/logger";import { executeWithRetry, attemptRequest } from "../lib/retry-utils";-dotenv.config();-const useFireEngine =- process.env.FIRE_ENGINE_BETA_URL !== "" &&- process.env.FIRE_ENGINE_BETA_URL !== undefined;+ config.FIRE_ENGINE_BETA_URL !== "" &&+ config.FIRE_ENGINE_BETA_URL !== undefined;function hasResults(results: unknown): results is SearchResult[] {return Array.isArray(results) && results.length > 0;@@ -41,7 +39,7 @@ export async function fire_engine_search(page: options.page ?? 1,};- const url = `${process.env.FIRE_ENGINE_BETA_URL}/search`;+ const url = `${config.FIRE_ENGINE_BETA_URL}/search`;const data = JSON.stringify(payload);const result = await executeWithRetry<SearchResult[]>(@@ -83,7 +81,7 @@ export async function fireEngineMap(page: options.page ?? 1,};- const url = `${process.env.FIRE_ENGINE_BETA_URL}/map`;+ const url = `${config.FIRE_ENGINE_BETA_URL}/map`;const data = JSON.stringify(payload);const result = await executeWithRetry<SearchResult[]>(diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.tsindex 40fd00d70..027724329 100644--- a/apps/api/src/search/index.ts+++ b/apps/api/src/search/index.ts@@ -1,4 +1,5 @@import { SearchResult } from "../../src/lib/entities";+import { config } from "../config";import { searxng_search } from "./searxng";import { fire_engine_search } from "./fireEngine";import { Logger } from "winston";@@ -32,7 +33,7 @@ export async function search({timeout?: number;}): Promise<SearchResult[]> {try {- if (process.env.FIRE_ENGINE_BETA_URL) {+ if (config.FIRE_ENGINE_BETA_URL) {logger.info("Using fire engine search");const results = await fire_engine_search(query, {numResults: num_results,@@ -44,7 +45,7 @@ export async function search({});return results;}- if (process.env.SEARXNG_ENDPOINT) {+ if (config.SEARXNG_ENDPOINT) {logger.info("Using searxng search");const results = await searxng_search(query, {num_results,diff --git a/apps/api/src/search/searxng.ts b/apps/api/src/search/searxng.tsindex 11d5a9e32..46017dfa0 100644--- a/apps/api/src/search/searxng.ts+++ b/apps/api/src/search/searxng.ts@@ -1,10 +1,8 @@import axios from "axios";-import dotenv from "dotenv";+import { config } from "../config";import { SearchResult } from "../../src/lib/entities";import { logger } from "../lib/logger";-dotenv.config();-interface SearchOptions {tbs?: string;filter?: string;@@ -25,13 +23,13 @@ export async function searxng_search(// gl: options.country, //not possible with SearXNG// location: options.location, //not possible with SearXNG// num: options.num_results, //not possible with SearXNG- engines: process.env.SEARXNG_ENGINES || "",- categories: process.env.SEARXNG_CATEGORIES || "",+ engines: config.SEARXNG_ENGINES ?? "",+ categories: config.SEARXNG_CATEGORIES ?? "",pageno: options.page ?? 1,format: "json",};- const url = process.env.SEARXNG_ENDPOINT!;+ const url = config.SEARXNG_ENDPOINT!;// Remove trailing slash if it existsconst cleanedUrl = url.endsWith("/") ? url.slice(0, -1) : url;diff --git a/apps/api/src/search/v2/ddgsearch.ts b/apps/api/src/search/v2/ddgsearch.tsindex 2f05cbb14..9838423ac 100644--- a/apps/api/src/search/v2/ddgsearch.ts+++ b/apps/api/src/search/v2/ddgsearch.ts@@ -1,4 +1,5 @@import * as undici from "undici";+import { config } from "../../config";import { JSDOM } from "jsdom";import { SearchV2Response, WebSearchResult } from "../../lib/entities";import { logger } from "../../lib/logger";@@ -224,7 +225,7 @@ export async function ddgSearch(return { web: results.slice(0, num_results) };} catch (error: any) {if (error instanceof DDGAntiBotError) {- if (process.env.TEST_SUITE_SELF_HOSTED) {+ if (config.TEST_SUITE_SELF_HOSTED) {logger.warn("DuckDuckGo: Blocked by anti-bot measures, returning dummy page for test suite...",{ term },diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex e89f53531..9ba5b1d86 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -1,4 +1,4 @@-import dotenv from "dotenv";+import { config } from "../../config";import {SearchResult,SearchV2Response,@@ -8,11 +8,9 @@ import * as Sentry from "@sentry/node";import { logger } from "../../lib/logger";import { executeWithRetry, attemptRequest } from "../../lib/retry-utils";-dotenv.config();-const useFireEngine =- process.env.FIRE_ENGINE_BETA_URL !== "" &&- process.env.FIRE_ENGINE_BETA_URL !== undefined;+ config.FIRE_ENGINE_BETA_URL !== "" &&+ config.FIRE_ENGINE_BETA_URL !== undefined;function normalizeSearchTypes(type?: SearchResultType | SearchResultType[],@@ -72,7 +70,7 @@ export async function fire_engine_search_v2(};const requestedTypes = normalizeSearchTypes(options.type);- const url = `${process.env.FIRE_ENGINE_BETA_URL}/v2/search`;+ const url = `${config.FIRE_ENGINE_BETA_URL}/v2/search`;const data = JSON.stringify(payload);const result = await executeWithRetry<SearchV2Response>(diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.tsindex 112ba5d09..3dca1a848 100644--- a/apps/api/src/search/v2/index.ts+++ b/apps/api/src/search/v2/index.ts@@ -1,4 +1,5 @@import { SearchV2Response, SearchResultType } from "../../lib/entities";+import { config } from "../../config";import { fire_engine_search_v2 } from "./fireEngine-v2";import { searxng_search } from "./searxng";import { ddgSearch } from "./ddgsearch";@@ -36,7 +37,7 @@ export async function search({enterprise?: ("default" | "anon" | "zdr")[];}): Promise<SearchV2Response> {try {- if (process.env.FIRE_ENGINE_BETA_URL) {+ if (config.FIRE_ENGINE_BETA_URL) {logger.info("Using fire engine search");const results = await fire_engine_search_v2(query, {numResults: num_results,@@ -52,7 +53,7 @@ export async function search({return results;}- if (process.env.SEARXNG_ENDPOINT) {+ if (config.SEARXNG_ENDPOINT) {logger.info("Using searxng search");const results = await searxng_search(query, {num_results,diff --git a/apps/api/src/search/v2/searxng.ts b/apps/api/src/search/v2/searxng.tsindex 550a35aec..99af9f9ea 100644--- a/apps/api/src/search/v2/searxng.ts+++ b/apps/api/src/search/v2/searxng.ts@@ -1,10 +1,8 @@import axios from "axios";-import dotenv from "dotenv";+import { config } from "../../config";import { SearchV2Response, WebSearchResult } from "../../lib/entities";import { logger } from "../../lib/logger";-dotenv.config();-interface SearchOptions {tbs?: string;filter?: string;@@ -25,13 +23,13 @@ export async function searxng_search(// gl: options.country, //not possible with SearXNG// location: options.location, //not possible with SearXNG// num: options.num_results, //not possible with SearXNG- engines: process.env.SEARXNG_ENGINES || "",- categories: process.env.SEARXNG_CATEGORIES || "",+ engines: config.SEARXNG_ENGINES ?? "",+ categories: config.SEARXNG_CATEGORIES ?? "",pageno: options.page ?? 1,format: "json",};- const url = process.env.SEARXNG_ENDPOINT!;+ const url = config.SEARXNG_ENDPOINT!;// Remove trailing slash if it existsconst cleanedUrl = url.endsWith("/") ? url.slice(0, -1) : url;diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.tsindex 9be7a6057..a9ac8f19f 100644--- a/apps/api/src/services/ab-test.ts+++ b/apps/api/src/services/ab-test.ts@@ -1,6 +1,7 @@import { ScrapeJobData } from "../types";import { logger as _logger } from "../lib/logger";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";+import { config } from "../config";import {FireEngineScrapeRequestChromeCDP,FireEngineScrapeRequestCommon,@@ -12,10 +13,9 @@ export function abTestJob(webScraperOptions: ScrapeJobData) {// Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATEconst abLogger = _logger.child({ method: "ABTestToStaging" });try {- const abRateEnv = process.env.SCRAPEURL_AB_RATE;- const abHostEnv = process.env.SCRAPEURL_AB_HOST;- const shouldExtendMaxAge =- process.env.SCRAPEURL_AB_EXTEND_MAXAGE === "true";+ const abRateEnv = config.SCRAPEURL_AB_RATE;+ const abHostEnv = config.SCRAPEURL_AB_HOST;+ const shouldExtendMaxAge = config.SCRAPEURL_AB_EXTEND_MAXAGE;const abRate =abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;const shouldABTest =@@ -82,8 +82,8 @@ export function abTestFireEngine(// Global A/B test: mirror request to staging fire-engine based on SCRAPEURL_AB_RATEconst abLogger = _logger.child({ method: "ABTestToStaging" });try {- const abRateEnv = process.env.FIRE_ENGINE_AB_RATE;- const abHostEnv = process.env.FIRE_ENGINE_AB_HOST;+ const abRateEnv = config.FIRE_ENGINE_AB_RATE;+ const abHostEnv = config.FIRE_ENGINE_AB_HOST;const abRate =abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;const shouldABTest =diff --git a/apps/api/src/services/agentLivecastWS.ts b/apps/api/src/services/agentLivecastWS.tsindex 4964ac83d..32af19d86 100644--- a/apps/api/src/services/agentLivecastWS.ts+++ b/apps/api/src/services/agentLivecastWS.ts@@ -1,4 +1,5 @@import { configDotenv } from "dotenv";+import { config } from "../config";import { logger } from "../lib/logger";import type { Request } from "express";import WSWebSocket from "ws";@@ -26,7 +27,7 @@ export function attachWsProxy(app: any) {const url = new URL(req.url ?? "", "http://placeholder/");const sessionIdParam = url.searchParams.get("userProvidedId") || "";- const workerWsUrl = `${process.env.FIRE_ENGINE_BETA_URL?.replace("http", "ws")}?userProvidedId=${sessionIdParam}`;+ const workerWsUrl = `${config.FIRE_ENGINE_BETA_URL?.replace("http", "ws")}?userProvidedId=${sessionIdParam}`;console.log(workerWsUrl);const wsWorker = new WebSocket(workerWsUrl);diff --git a/apps/api/src/services/alerts/slack.ts b/apps/api/src/services/alerts/slack.tsindex ad8f9186a..cfee2aafd 100644--- a/apps/api/src/services/alerts/slack.ts+++ b/apps/api/src/services/alerts/slack.ts@@ -1,10 +1,11 @@import axios from "axios";import { logger } from "../../../src/lib/logger";+import { config } from "../../config";export async function sendSlackWebhook(message: string,alertEveryone: boolean = false,- webhookUrl: string = process.env.SLACK_WEBHOOK_URL ?? "",+ webhookUrl: string = config.SLACK_WEBHOOK_URL ?? "",) {const messagePrefix = alertEveryone ? "<!channel> " : "";const payload = {diff --git a/apps/api/src/services/billing/auto_charge.ts b/apps/api/src/services/billing/auto_charge.tsindex 57ca143d7..735871006 100644--- a/apps/api/src/services/billing/auto_charge.ts+++ b/apps/api/src/services/billing/auto_charge.ts@@ -1,5 +1,6 @@// Import necessary dependencies and typesimport { AuthCreditUsageChunk } from "../../controllers/v1/types";+import { config } from "../../config";import { clearACUC, clearACUCTeam, getACUC } from "../../controllers/auth";import { redlock } from "../redlock";import { supabase_rr_service, supabase_service } from "../supabase";@@ -332,11 +333,11 @@ async function _autoChargeScale(logger.info("Scale auto-recharge successful");- if (process.env.SLACK_ADMIN_WEBHOOK_URL) {+ if (config.SLACK_ADMIN_WEBHOOK_URL) {sendSlackWebhook(`💰 Auto-recharge successful on team ${chunk.team_id} for ${price.credits} credits (total auto-recharges this month: ${rechargesThisMonth.length + 1}).`,false,- process.env.SLACK_ADMIN_WEBHOOK_URL,+ config.SLACK_ADMIN_WEBHOOK_URL,).catch(error => {logger.debug(`Error sending slack notification: ${error}`);});@@ -580,7 +581,7 @@ async function _autoChargeSelfServe(paymentStatus: paymentStatus.return_status,});- if (process.env.SLACK_ADMIN_WEBHOOK_URL) {+ if (config.SLACK_ADMIN_WEBHOOK_URL) {const webhookCooldownKey = `webhook_cooldown_${chunk.team_id}`;const isInCooldown = await getValue(webhookCooldownKey);@@ -588,7 +589,7 @@ async function _autoChargeSelfServe(sendSlackWebhook(`Auto-recharge: Team ${chunk.team_id}. ${AUTO_RECHARGE_CREDITS} credits added. Payment status: ${paymentStatus.return_status}.`,false,- process.env.SLACK_ADMIN_WEBHOOK_URL,+ config.SLACK_ADMIN_WEBHOOK_URL,).catch(error => {logger.debug(`Error sending slack notification: ${error}`,diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.tsindex b73d2b67a..8878c2145 100644--- a/apps/api/src/services/billing/batch_billing.ts+++ b/apps/api/src/services/billing/batch_billing.ts@@ -1,4 +1,5 @@import { logger } from "../../lib/logger";+import { config } from "../../config";import { getRedisConnection } from "../queue-service";import { supabase_service } from "../supabase";import * as Sentry from "@sentry/node";@@ -241,7 +242,7 @@ export async function queueBillingOperation(// Should we add this?// I guess batch is fast enough that it's fine- // if (process.env.USE_DB_AUTHENTICATION === "true") {+ // if (config.USE_DB_AUTHENTICATION) {// (async () => {// // Get API keys for this team to update in cache// const { data } = await supabase_servicediff --git a/apps/api/src/services/billing/stripe.ts b/apps/api/src/services/billing/stripe.tsindex 8a43e53ca..4e3444da0 100644--- a/apps/api/src/services/billing/stripe.ts+++ b/apps/api/src/services/billing/stripe.ts@@ -1,7 +1,8 @@import { logger } from "../../lib/logger";+import { config } from "../../config";import Stripe from "stripe";-const stripe = new Stripe(process.env.STRIPE_SECRET_KEY ?? "");+const stripe = new Stripe(config.STRIPE_SECRET_KEY ?? "");async function getCustomerDefaultPaymentMethod(customerId: string) {const paymentMethods = await stripe.customers.listPaymentMethods(customerId, {diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex f57739344..007705d22 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -1,4 +1,5 @@import "dotenv/config";+import { config } from "../config";import "./sentry";import { setSentryServiceTag } from "./sentry";import * as Sentry from "@sentry/node";@@ -26,16 +27,12 @@ configDotenv();const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));-const jobLockExtendInterval =- Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;-const jobLockExtensionTime =- Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;+const jobLockExtendInterval = config.JOB_LOCK_EXTEND_INTERVAL;+const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;-const cantAcceptConnectionInterval =- Number(process.env.CANT_ACCEPT_CONNECTION_INTERVAL) || 2000;-const connectionMonitorInterval =- Number(process.env.CONNECTION_MONITOR_INTERVAL) || 10;-const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;+const cantAcceptConnectionInterval = config.CANT_ACCEPT_CONNECTION_INTERVAL;+const connectionMonitorInterval = config.CONNECTION_MONITOR_INTERVAL;+const gotJobInterval = config.CONNECTION_MONITOR_INTERVAL;const runningJobs: Set<string> = new Set();@@ -259,11 +256,11 @@ let currentLiveness: boolean = true;app.get("/liveness", (req, res) => {_logger.info("Liveness endpoint hit");- if (process.env.USE_DB_AUTHENTICATION === "true") {+ if (config.USE_DB_AUTHENTICATION) {// networking check for Kubernetes environments- const host = process.env.FIRECRAWL_APP_HOST || "firecrawl-app-service";- const port = process.env.FIRECRAWL_APP_PORT || "3002";- const scheme = process.env.FIRECRAWL_APP_SCHEME || "http";+ const host = config.FIRECRAWL_APP_HOST;+ const port = config.FIRECRAWL_APP_PORT;+ const scheme = config.FIRECRAWL_APP_SCHEME;robustFetch({url: `${scheme}://${host}:${port}`,@@ -289,7 +286,7 @@ app.get("/liveness", (req, res) => {}});-const workerPort = process.env.EXTRACT_WORKER_PORT || process.env.PORT || 3005;+const workerPort = config.EXTRACT_WORKER_PORT || config.PORT;app.listen(workerPort, () => {_logger.info(`Liveness endpoint is running on port ${workerPort}`);});diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex c262dfc99..4d4f54cc7 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -10,6 +10,7 @@ import { MapDocument } from "../controllers/v2/types";import { PdfMetadata } from "@mendable/firecrawl-rs";import { storage } from "../lib/gcs-jobs";import { withSpan, setSpanAttributes } from "../lib/otel-tracer";+import { config } from "../config";configDotenv();// SupabaseService class initializes the Supabase client conditionally based on environment variables.@@ -17,8 +18,8 @@ class IndexSupabaseService {private client: SupabaseClient | null = null;constructor() {- const supabaseUrl = process.env.INDEX_SUPABASE_URL;- const supabaseServiceToken = process.env.INDEX_SUPABASE_SERVICE_TOKEN;+ const supabaseUrl = config.INDEX_SUPABASE_URL;+ const supabaseServiceToken = config.INDEX_SUPABASE_SERVICE_TOKEN;// Only initialize the Supabase client if both URL and Service Token are provided.if (!supabaseUrl || !supabaseServiceToken) {// Warn the user that Authentication is disabled by setting the client to null@@ -57,8 +58,8 @@ export const index_supabase_service: SupabaseClient = new Proxy(serv, {},}) as unknown as SupabaseClient;-const credentials = process.env.GCS_CREDENTIALS- ? JSON.parse(atob(process.env.GCS_CREDENTIALS))+const credentials = config.GCS_CREDENTIALS+ ? JSON.parse(atob(config.GCS_CREDENTIALS)): undefined;export async function getIndexFromGCS(@@ -72,12 +73,12 @@ export async function getIndexFromGCS("index.url": url,});- if (!process.env.GCS_INDEX_BUCKET_NAME) {+ if (!config.GCS_INDEX_BUCKET_NAME) {setSpanAttributes(span, { "gcs.index_bucket_configured": false });return null;}- const bucket = storage.bucket(process.env.GCS_INDEX_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_INDEX_BUCKET_NAME);const blob = bucket.file(`${url}`);const [blobContent] = await blob.download();const parsed = JSON.parse(blobContent.toString());@@ -106,7 +107,7 @@ export async function getIndexFromGCS() {logger?.info("Re-signing screenshot URL");const [url] = await storage- .bucket(process.env.GCS_MEDIA_BUCKET_NAME!)+ .bucket(config.GCS_MEDIA_BUCKET_NAME!).file(decodeURIComponent(screenshotUrl.pathname.split("/")[2])).getSignedUrl({action: "read",@@ -169,12 +170,12 @@ export async function saveIndexToGCS("index.has_error": !!doc.error,});- if (!process.env.GCS_INDEX_BUCKET_NAME) {+ if (!config.GCS_INDEX_BUCKET_NAME) {setSpanAttributes(span, { "gcs.index_bucket_configured": false });return;}- const bucket = storage.bucket(process.env.GCS_INDEX_BUCKET_NAME);+ const bucket = storage.bucket(config.GCS_INDEX_BUCKET_NAME);const blob = bucket.file(`${id}.json`);for (let i = 0; i < 3; i++) {@@ -200,12 +201,11 @@ export async function saveIndexToGCS(}export const useIndex =- process.env.INDEX_SUPABASE_URL !== "" &&- process.env.INDEX_SUPABASE_URL !== undefined;+ config.INDEX_SUPABASE_URL !== "" && config.INDEX_SUPABASE_URL !== undefined;export const useSearchIndex =- process.env.SEARCH_INDEX_SUPABASE_URL !== "" &&- process.env.SEARCH_INDEX_SUPABASE_URL !== undefined;+ config.SEARCH_INDEX_SUPABASE_URL !== "" &&+ config.SEARCH_INDEX_SUPABASE_URL !== undefined;export function normalizeURLForIndex(url: string): string {const urlObj = new URL(url);@@ -664,7 +664,7 @@ export async function queryIndexAtSplitLevel(limit: number,maxAge = 2 * 24 * 60 * 60 * 1000,): Promise<string[]> {- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {return [];}@@ -717,7 +717,7 @@ export async function queryIndexAtDomainSplitLevel(limit: number,maxAge = 2 * 24 * 60 * 60 * 1000,): Promise<string[]> {- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {return [];}@@ -769,7 +769,7 @@ export async function queryOMCESignatures(hostname: string,maxAge = 2 * 24 * 60 * 60 * 1000,): Promise<string[]> {- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {return [];}@@ -800,7 +800,7 @@ export async function queryIndexAtSplitLevelWithMeta(url: string,limit: number,): Promise<MapDocument[]> {- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {return [];}@@ -860,7 +860,7 @@ export async function queryIndexAtDomainSplitLevelWithMeta(hostname: string,limit: number,): Promise<MapDocument[]> {- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {return [];}@@ -928,7 +928,7 @@ export async function queryDomainsForPrecrawl(maxDomains = 50,logger: Logger = _logger,): Promise<DomainPriority[]> {- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {return [];}diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex ab58050eb..cdb39998b 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -1,4 +1,5 @@import "dotenv/config";+import { config } from "../../config";import "../sentry";import { setSentryServiceTag } from "../sentry";import * as Sentry from "@sentry/node";@@ -42,19 +43,14 @@ import { getACUCTeam } from "../../controllers/auth";import { supabase_service } from "../supabase";import { processEngpickerJob } from "../../lib/engpicker";-const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;-const workerStalledCheckInterval =- Number(process.env.WORKER_STALLED_CHECK_INTERVAL) || 30000;-const jobLockExtendInterval =- Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 15000;-const jobLockExtensionTime =- Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;+const workerLockDuration = config.WORKER_LOCK_DURATION;+const workerStalledCheckInterval = config.WORKER_STALLED_CHECK_INTERVAL;+const jobLockExtendInterval = config.JOB_LOCK_EXTEND_INTERVAL;+const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;-const cantAcceptConnectionInterval =- Number(process.env.CANT_ACCEPT_CONNECTION_INTERVAL) || 2000;-const connectionMonitorInterval =- Number(process.env.CONNECTION_MONITOR_INTERVAL) || 10;-const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;+const cantAcceptConnectionInterval = config.CANT_ACCEPT_CONNECTION_INTERVAL;+const connectionMonitorInterval = config.CONNECTION_MONITOR_INTERVAL;+const gotJobInterval = config.CONNECTION_MONITOR_INTERVAL;const runningJobs: Set<string> = new Set();@@ -150,7 +146,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {const MIN_URLS_PER_DOMAIN = 10;const MAX_URLS_PER_DOMAIN = 250;- const teamId = process.env.PRECRAWL_TEAM_ID;+ const teamId = config.PRECRAWL_TEAM_ID;try {await withSpan("precrawl.job", async span => {@@ -470,8 +466,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {internalOptions: {disableSmartWaitCache: true, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matterteamId,- saveScrapeResultToGCS:- !!process.env.GCS_FIRE_ENGINE_BUCKET_NAME,+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,zeroDataRetention: false,isPreCrawl: true, // NOTE: must be added to internal options for indexing, if not it will be treated as a normal scrape in the index},@@ -692,7 +687,7 @@ const DOMAIN_FREQUENCY_INTERVAL = 10000;processBillingJobInternal,);- const precrawlWorkerPromise = process.env.PRECRAWL_TEAM_ID+ const precrawlWorkerPromise = config.PRECRAWL_TEAM_ID? workerFun(getPrecrawlQueue(), processPrecrawlJob): (async () => {logger.warn("PRECRAWL_TEAM_ID not set, skipping precrawl worker");diff --git a/apps/api/src/services/ledger/supabase-ledger.ts b/apps/api/src/services/ledger/supabase-ledger.tsindex 434214910..7767aded2 100644--- a/apps/api/src/services/ledger/supabase-ledger.ts+++ b/apps/api/src/services/ledger/supabase-ledger.ts@@ -1,6 +1,7 @@import { createClient, SupabaseClient } from "@supabase/supabase-js";import { logger } from "../../lib/logger";import { configDotenv } from "dotenv";+import { config } from "../../config";configDotenv();// SupabaseLedgerService class initializes the Supabase client for the ledger schema@@ -9,10 +10,10 @@ class SupabaseLedgerService {private rrClient: SupabaseClient<any, "ledger", any> | null = null;constructor() {- const supabaseUrl = process.env.SUPABASE_URL;- const supabaseReplicaUrl = process.env.SUPABASE_REPLICA_URL;- const supabaseServiceToken = process.env.SUPABASE_SERVICE_TOKEN;- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";+ const supabaseUrl = config.SUPABASE_URL;+ const supabaseReplicaUrl = config.SUPABASE_REPLICA_URL;+ const supabaseServiceToken = config.SUPABASE_SERVICE_TOKEN;+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;// Only initialize the Supabase client if both URL and Service Token are provided.if (!useDbAuthentication) {diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 0515e37b7..4361498a4 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -1,4 +1,5 @@import { supabase_service } from "../supabase";+import { config } from "../../config";import "dotenv/config";import { logger as _logger } from "../../lib/logger";import { configDotenv } from "dotenv";@@ -24,7 +25,7 @@ async function robustInsert(force: boolean,logger: Logger,) {- if (process.env.USE_DB_AUTHENTICATION !== "true") {+ if (config.USE_DB_AUTHENTICATION !== true) {logger.info("Skipping database insertion due to USE_DB_AUTHENTICATION being off",{ table },@@ -193,7 +194,7 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {if (scrape.doc &&- process.env.GCS_BUCKET_NAME &&+ config.GCS_BUCKET_NAME &&!(scrape.skipNuq && scrape.zeroDataRetention)) {await saveScrapeToGCS(scrape);@@ -202,7 +203,7 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {if (scrape.is_successful &&!scrape.zeroDataRetention &&- process.env.USE_DB_AUTHENTICATION === "true"+ config.USE_DB_AUTHENTICATION) {const hasMarkdown = hasFormatOfType(scrape.options.formats, "markdown");const hasChangeTracking = hasFormatOfType(diff --git a/apps/api/src/services/notification/email_notification.ts b/apps/api/src/services/notification/email_notification.tsindex 303efbb42..4f7f871d5 100644--- a/apps/api/src/services/notification/email_notification.ts+++ b/apps/api/src/services/notification/email_notification.ts@@ -1,4 +1,5 @@import { supabase_service } from "../supabase";+import { config } from "../../config";import { withAuth } from "../../lib/withAuth";import { Resend } from "resend";import { NotificationType } from "../../types";@@ -104,7 +105,7 @@ async function sendEmailNotification(notificationType: NotificationType,context: NotificationContext = {},) {- const resend = new Resend(process.env.RESEND_API_KEY);+ const resend = new Resend(config.RESEND_API_KEY);try {// Get user's email preferences@@ -297,11 +298,11 @@ async function sendNotificationInternal(},]);- if (process.env.SLACK_ADMIN_WEBHOOK_URL && emails.length > 0) {+ if (config.SLACK_ADMIN_WEBHOOK_URL && emails.length > 0) {sendSlackWebhook(`${getNotificationString(notificationType)}: Team ${team_id}, with email ${emails[0].email}. Number of credits used: ${chunk.adjusted_credits_used} | Number of credits in the plan: ${chunk.price_credits}`,false,- process.env.SLACK_ADMIN_WEBHOOK_URL,+ config.SLACK_ADMIN_WEBHOOK_URL,).catch(error => {logger.debug(`Error sending slack notification: ${error}`);});@@ -428,14 +429,14 @@ export async function sendNotificationWithCustomDays(]);if (- process.env.SLACK_ADMIN_WEBHOOK_URL &&+ config.SLACK_ADMIN_WEBHOOK_URL &&emails.length > 0 &¬ificationType !== NotificationType.CONCURRENCY_LIMIT_REACHED) {sendSlackWebhook(`${getNotificationString(notificationType)}: Team ${team_id}, with email ${emails[0].email}.`,false,- process.env.SLACK_ADMIN_WEBHOOK_URL,+ config.SLACK_ADMIN_WEBHOOK_URL,).catch(error => {logger.debug(`Error sending slack notification: ${error}`);});diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 6bd80afe6..465ec6761 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -1,4 +1,5 @@import { Queue } from "bullmq";+import { config } from "../config";import { logger } from "../lib/logger";import IORedis from "ioredis";import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";@@ -14,7 +15,7 @@ let redisConnection: IORedis;export function getRedisConnection(): IORedis {if (!redisConnection) {- redisConnection = new IORedis(process.env.REDIS_URL!, {+ redisConnection = new IORedis(config.REDIS_URL!, {maxRetriesPerRequest: null,});redisConnection.on("connect", () => logger.info("Redis connected"));diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 53e9cd227..3bda3f5b3 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -1,4 +1,5 @@import "dotenv/config";+import { config } from "../config";import "./sentry";import { setSentryServiceTag } from "./sentry";import * as Sentry from "@sentry/node";@@ -29,16 +30,12 @@ configDotenv();const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));-const jobLockExtendInterval =- Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;-const jobLockExtensionTime =- Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;+const jobLockExtendInterval = config.JOB_LOCK_EXTEND_INTERVAL;+const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;-const cantAcceptConnectionInterval =- Number(process.env.CANT_ACCEPT_CONNECTION_INTERVAL) || 2000;-const connectionMonitorInterval =- Number(process.env.CONNECTION_MONITOR_INTERVAL) || 10;-const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;+const cantAcceptConnectionInterval = config.CANT_ACCEPT_CONNECTION_INTERVAL;+const connectionMonitorInterval = config.CONNECTION_MONITOR_INTERVAL;+const gotJobInterval = config.CONNECTION_MONITOR_INTERVAL;const runningJobs: Set<string> = new Set();@@ -328,7 +325,7 @@ const crawlFinishWorker = async () => {if (job === null) {__logger.info("No jobs to process", { module: "nuq/metrics" });await new Promise(resolve => setTimeout(resolve, noJobTimeout));- if (!process.env.NUQ_RABBITMQ_URL) {+ if (!config.NUQ_RABBITMQ_URL) {noJobTimeout = Math.min(noJobTimeout * 2, 10000);}continue;@@ -408,11 +405,11 @@ let currentLiveness: boolean = true;app.get("/liveness", (req, res) => {_logger.info("Liveness endpoint hit");- if (process.env.USE_DB_AUTHENTICATION === "true") {+ if (config.USE_DB_AUTHENTICATION) {// networking check for Kubernetes environments- const host = process.env.FIRECRAWL_APP_HOST || "firecrawl-app-service";- const port = process.env.FIRECRAWL_APP_PORT || "3002";- const scheme = process.env.FIRECRAWL_APP_SCHEME || "http";+ const host = config.FIRECRAWL_APP_HOST;+ const port = config.FIRECRAWL_APP_PORT;+ const scheme = config.FIRECRAWL_APP_SCHEME;robustFetch({url: `${scheme}://${host}:${port}`,@@ -438,7 +435,7 @@ app.get("/liveness", (req, res) => {}});-const workerPort = process.env.WORKER_PORT || process.env.PORT || 3005;+const workerPort = config.WORKER_PORT || config.PORT;app.listen(workerPort, () => {_logger.info(`Liveness endpoint is running on port ${workerPort}`);});diff --git a/apps/api/src/services/rate-limiter.test.ts b/apps/api/src/services/rate-limiter.test.tsindex 5e8e067f2..709b32904 100644--- a/apps/api/src/services/rate-limiter.test.ts+++ b/apps/api/src/services/rate-limiter.test.ts@@ -1,3 +1,4 @@+import { config } from "../config";// import {// getRateLimiter,// serverRateLimiter,@@ -10,7 +11,7 @@// beforeAll(async () => {// try {// await redisRateLimitClient.connect();-// // if (process.env.REDIS_RATE_LIMIT_URL === "redis://localhost:6379") {+// // if (config.REDIS_RATE_LIMIT_URL === "redis://localhost:6379") {// // console.log("Erasing all keys");// // // erase all the keys that start with "test-prefix"// // const keys = await redisRateLimitClient.keys("test-prefix:*");@@ -23,7 +24,7 @@// afterAll(async () => {// try {-// // if (process.env.REDIS_RATE_LIMIT_URL === "redis://localhost:6379") {+// // if (config.REDIS_RATE_LIMIT_URL === "redis://localhost:6379") {// await redisRateLimitClient.disconnect();// // }// } catch (error) {}diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.tsindex 2b6bbda05..deea0e867 100644--- a/apps/api/src/services/rate-limiter.ts+++ b/apps/api/src/services/rate-limiter.ts@@ -1,11 +1,10 @@import { RateLimiterRedis } from "rate-limiter-flexible";+import { config } from "../config";import { RateLimiterMode } from "../types";import Redis from "ioredis";import type { AuthCreditUsageChunk } from "../controllers/v1/types";-export const redisRateLimitClient = new Redis(- process.env.REDIS_RATE_LIMIT_URL!,-);+export const redisRateLimitClient = new Redis(config.REDIS_RATE_LIMIT_URL!);const createRateLimiter = (keyPrefix, points) =>new RateLimiterRedis({diff --git a/apps/api/src/services/redis.ts b/apps/api/src/services/redis.tsindex af5f9f7ca..7049020c0 100644--- a/apps/api/src/services/redis.ts+++ b/apps/api/src/services/redis.ts@@ -1,4 +1,5 @@import IORedis from "ioredis";+import { config } from "../config";import { redisRateLimitClient } from "./rate-limiter";import { logger } from "../lib/logger";@@ -71,6 +72,5 @@ const deleteKey = async (key: string) => {export { setValue, getValue, deleteKey };-const redisEvictURL =- process.env.REDIS_EVICT_URL ?? process.env.REDIS_RATE_LIMIT_URL;+const redisEvictURL = config.REDIS_EVICT_URL ?? config.REDIS_RATE_LIMIT_URL;export const redisEvictConnection = new IORedis(redisEvictURL!);diff --git a/apps/api/src/services/redlock.ts b/apps/api/src/services/redlock.tsindex 923cfc3d0..25a9767ce 100644--- a/apps/api/src/services/redlock.ts+++ b/apps/api/src/services/redlock.ts@@ -1,10 +1,11 @@import Redlock from "redlock";+import { config } from "../config";import Client from "ioredis";export const redlock = new Redlock(// You should have one client for each independent redis node// or cluster.- [new Client(process.env.REDIS_RATE_LIMIT_URL!)],+ [new Client(config.REDIS_RATE_LIMIT_URL!)],{// The expected clock drift; for more details see:// http://redis.io/topics/distlockdiff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.tsindex 1c4a7fecd..d7e51838c 100644--- a/apps/api/src/services/sentry.ts+++ b/apps/api/src/services/sentry.ts@@ -1,18 +1,12 @@import * as Sentry from "@sentry/node";import { logger } from "../lib/logger";+import { config } from "../config";-if (process.env.SENTRY_DSN) {+if (config.SENTRY_DSN) {logger.info("Setting up Sentry...");- const TRACE_SAMPLE_RATE = parseFloat(- process.env.SENTRY_TRACE_SAMPLE_RATE || "0",- );- const ERROR_SAMPLE_RATE = parseFloat(- process.env.SENTRY_ERROR_SAMPLE_RATE || "0.05",- );-Sentry.init({- dsn: process.env.SENTRY_DSN,+ dsn: config.SENTRY_DSN,integrations: integrations => [...integrations,Sentry.vercelAIIntegration({@@ -22,11 +16,13 @@ if (process.env.SENTRY_DSN) {],tracesSampler: samplingContext => {// trace all AI spans, sample 1% of all others- return samplingContext.name?.startsWith("ai.") ? 1.0 : TRACE_SAMPLE_RATE;+ return samplingContext.name?.startsWith("ai.")+ ? 1.0+ : config.SENTRY_TRACE_SAMPLE_RATE;},- sampleRate: ERROR_SAMPLE_RATE,- serverName: process.env.NUQ_POD_NAME,- environment: process.env.SENTRY_ENVIRONMENT ?? "production",+ sampleRate: config.SENTRY_ERROR_SAMPLE_RATE,+ serverName: config.NUQ_POD_NAME,+ environment: config.SENTRY_ENVIRONMENT,beforeSend(event, hint) {const error = hint?.originalException;@@ -70,7 +66,7 @@ if (process.env.SENTRY_DSN) {* This helps distinguish between API server and worker errors in Sentry*/export function setSentryServiceTag(serviceType: string) {- if (process.env.SENTRY_DSN) {+ if (config.SENTRY_DSN) {Sentry.setTag("service_type", serviceType);}}diff --git a/apps/api/src/services/supabase.ts b/apps/api/src/services/supabase.tsindex 62a08baf1..71dc4991d 100644--- a/apps/api/src/services/supabase.ts+++ b/apps/api/src/services/supabase.ts@@ -1,4 +1,5 @@import { createClient, SupabaseClient } from "@supabase/supabase-js";+import { config } from "../config";import { logger } from "../lib/logger";import { configDotenv } from "dotenv";configDotenv();@@ -11,16 +12,15 @@ class SupabaseService {private acucClient: SupabaseClient | null = null;constructor() {- const supabaseUrl = process.env.SUPABASE_URL;- const supabaseReplicaUrl = process.env.SUPABASE_REPLICA_URL;- const supabaseServiceToken = process.env.SUPABASE_SERVICE_TOKEN;- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";+ const supabaseUrl = config.SUPABASE_URL;+ const supabaseReplicaUrl = config.SUPABASE_REPLICA_URL;+ const supabaseServiceToken = config.SUPABASE_SERVICE_TOKEN;+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;// Only initialize the Supabase client if both URL and Service Token are provided.if (!useDbAuthentication) {- if (!!process.env.SUPABASE_ACUC_URL) {- const supabaseAcucUrl = process.env.SUPABASE_ACUC_URL;- const supabaseAcucServiceToken =- process.env.SUPABASE_ACUC_SERVICE_TOKEN;+ if (!!config.SUPABASE_ACUC_URL) {+ const supabaseAcucUrl = config.SUPABASE_ACUC_URL;+ const supabaseAcucServiceToken = config.SUPABASE_ACUC_SERVICE_TOKEN;if (!supabaseAcucUrl || !supabaseAcucServiceToken) {logger.error("Supabase ACUC environment variables aren't configured correctly. Supabase ACUC client will not be initialized. Fix ENV configuration or disable ACUC with USE_DB_AUTHENTICATION env variable",diff --git a/apps/api/src/services/system-monitor.ts b/apps/api/src/services/system-monitor.tsindex 23f6d396b..afe807cae 100644--- a/apps/api/src/services/system-monitor.ts+++ b/apps/api/src/services/system-monitor.ts@@ -1,16 +1,15 @@import si from "systeminformation";+import { config } from "../config";import { Mutex } from "async-mutex";import os from "os";import fs from "fs";import { logger } from "../lib/logger";-const IS_KUBERNETES = process.env.IS_KUBERNETES === "true";+const IS_KUBERNETES = config.IS_KUBERNETES;-const MAX_CPU = process.env.MAX_CPU ? parseFloat(process.env.MAX_CPU) : 0.8;-const MAX_RAM = process.env.MAX_RAM ? parseFloat(process.env.MAX_RAM) : 0.8;-const CACHE_DURATION = process.env.SYS_INFO_MAX_CACHE_DURATION- ? parseFloat(process.env.SYS_INFO_MAX_CACHE_DURATION)- : 150;+const MAX_CPU = config.MAX_CPU;+const MAX_RAM = config.MAX_RAM;+const CACHE_DURATION = config.SYS_INFO_MAX_CACHE_DURATION;class SystemMonitor {private static instance: SystemMonitor;diff --git a/apps/api/src/services/webhook/config.ts b/apps/api/src/services/webhook/config.tsindex 36f1dc784..5b35a4314 100644--- a/apps/api/src/services/webhook/config.ts+++ b/apps/api/src/services/webhook/config.ts@@ -2,6 +2,7 @@ import { logger as _logger } from "../../lib/logger";import { supabase_rr_service } from "../supabase";import { WebhookConfig } from "./types";+import { config } from "../../config";export async function getWebhookConfig(teamId: string,jobId: string,@@ -15,7 +16,7 @@ export async function getWebhookConfig(return { config: webhook, secret: await getHmacSecret(teamId) };}- const selfHostedUrl = process.env.SELF_HOSTED_WEBHOOK_URL?.replace(+ const selfHostedUrl = config.SELF_HOSTED_WEBHOOK_URL?.replace("{{JOB_ID}}",jobId,);@@ -27,11 +28,11 @@ export async function getWebhookConfig(metadata: {},events: ["completed", "failed", "page", "started"],},- secret: process.env.SELF_HOSTED_WEBHOOK_HMAC_SECRET,+ secret: config.SELF_HOSTED_WEBHOOK_HMAC_SECRET,};}- if (process.env.USE_DB_AUTHENTICATION === "true") {+ if (config.USE_DB_AUTHENTICATION) {const dbConfig = await fetchWebhookFromDb(teamId);if (dbConfig) {return { config: dbConfig, secret: await getHmacSecret(teamId) };@@ -59,8 +60,8 @@ async function fetchWebhookFromDb(}async function getHmacSecret(teamId: string): Promise<string | undefined> {- if (process.env.USE_DB_AUTHENTICATION !== "true") {- return process.env.SELF_HOSTED_WEBHOOK_HMAC_SECRET;+ if (config.USE_DB_AUTHENTICATION !== true) {+ return config.SELF_HOSTED_WEBHOOK_HMAC_SECRET;}try {diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.tsindex 4fe61e6f6..f91ac0d54 100644--- a/apps/api/src/services/webhook/delivery.ts+++ b/apps/api/src/services/webhook/delivery.ts@@ -1,4 +1,5 @@import undici from "undici";+import { config } from "../../config";import { createHmac } from "crypto";import { logger as _logger, logger } from "../../lib/logger";import {@@ -62,7 +63,7 @@ export class WebhookSender {}private shouldSendEvent(event: WebhookEvent): boolean {- if (process.env.DISABLE_WEBHOOK_DELIVERY === "true") {+ if (config.DISABLE_WEBHOOK_DELIVERY) {return false;}@@ -76,10 +77,7 @@ export class WebhookSender {private async deliver(payload: any, scrapeId?: string): Promise<void> {const webhookHost = new URL(this.config.url).hostname;- if (- isIPPrivate(webhookHost) &&- process.env.ALLOW_LOCAL_WEBHOOKS !== "true"- ) {+ if (isIPPrivate(webhookHost) && config.ALLOW_LOCAL_WEBHOOKS !== true) {this.logger.warn("Aborting webhook call to private IP address", {webhookUrl: this.config.url,});diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex 06dd1960a..35516137f 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -1,4 +1,5 @@import { logger as _logger } from "../../lib/logger";+import { config } from "../../config";import {finishCrawl,getCrawlJobs,@@ -128,7 +129,7 @@ export async function finishCrawlSuper(job: NuQJob<any>) {let credits_billed = null;- if (process.env.USE_DB_AUTHENTICATION === "true") {+ if (config.USE_DB_AUTHENTICATION) {const creditsRpc = await supabase_service.rpc("credits_billed_by_crawl_id_2",{diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.tsindex c2f329a49..909bd10f4 100644--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts@@ -1,4 +1,5 @@import "dotenv/config";+import { config } from "../../config";import "../sentry";import { setSentryServiceTag } from "../sentry";import {@@ -27,12 +28,9 @@ import { logger } from "../../lib/logger";}});- const server = app.listen(- Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011),- () => {- logger.info("NuQ prefetch worker metrics server started");- },- );+ const server = app.listen(config.NUQ_PREFETCH_WORKER_PORT, () => {+ logger.info("NuQ prefetch worker metrics server started");+ });async function shutdown() {server.close();diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.tsindex 883f0ee50..cb4aaf0ec 100644--- a/apps/api/src/services/worker/nuq-worker.ts+++ b/apps/api/src/services/worker/nuq-worker.ts@@ -1,4 +1,5 @@import "dotenv/config";+import { config } from "../../config";import "../sentry";import { setSentryServiceTag } from "../sentry";import { logger as _logger } from "../../lib/logger";@@ -37,12 +38,9 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f}});- const server = app.listen(- Number(process.env.NUQ_WORKER_PORT ?? process.env.PORT ?? 3000),- () => {- _logger.info("NuQ worker metrics server started");- },- );+ const server = app.listen(config.NUQ_WORKER_PORT, () => {+ _logger.info("NuQ worker metrics server started");+ });function shutdown() {isShuttingDown = true;@@ -59,7 +57,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-fif (job === null) {_logger.info("No jobs to process", { module: "nuq/metrics" });await new Promise(resolve => setTimeout(resolve, noJobTimeout));- if (!process.env.NUQ_RABBITMQ_URL) {+ if (!config.NUQ_RABBITMQ_URL) {noJobTimeout = Math.min(noJobTimeout * 2, 10000);}continue;diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 6f737ed5f..12e366c11 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -5,11 +5,12 @@ import { type ScrapeJobData } from "../../types";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import amqp from "amqplib";import { v5 as uuidv5, validate as isUUID } from "uuid";+import { config } from "../../config";// === Basicsconst nuqPool = new Pool({- connectionString: process.env.NUQ_DATABASE_URL, // may be a pgbouncer transaction pooler URL+ connectionString: config.NUQ_DATABASE_URL, // may be a pgbouncer transaction pooler URLapplication_name: "nuq",});@@ -63,7 +64,7 @@ function normalizeOwnerId(ownerId: string | undefined | null): string | null {class NuQ<JobData = any, JobReturnValue = any> {private listenChannelId: string =- (process.env.NUQ_POD_NAME ?? "main") + "-" + crypto.randomUUID();+ config.NUQ_POD_NAME + "-" + crypto.randomUUID();constructor(public readonly queueName: string,@@ -93,11 +94,11 @@ class NuQ<JobData = any, JobReturnValue = any> {private async startListener() {if (this.listener || this.shuttingDown || this.listenerStarting) return;- if (process.env.NUQ_RABBITMQ_URL) {+ if (config.NUQ_RABBITMQ_URL) {this.listenerStarting = true;try {- const connection = await amqp.connect(process.env.NUQ_RABBITMQ_URL);+ const connection = await amqp.connect(config.NUQ_RABBITMQ_URL);const channel = await connection.createChannel();await channel.prefetch(1);const queue = await channel.assertQueue(@@ -184,7 +185,7 @@ class NuQ<JobData = any, JobReturnValue = any> {type: "postgres",client: new Client({connectionString:- process.env.NUQ_DATABASE_URL_LISTEN ?? process.env.NUQ_DATABASE_URL, // will always be a direct connection+ config.NUQ_DATABASE_URL_LISTEN ?? config.NUQ_DATABASE_URL, // will always be a direct connectionapplication_name: "nuq_listener",}),};@@ -267,8 +268,8 @@ class NuQ<JobData = any, JobReturnValue = any> {private async startSender() {if (this.sender || this.shuttingDown) return;- if (process.env.NUQ_RABBITMQ_URL) {- const connection = await amqp.connect(process.env.NUQ_RABBITMQ_URL);+ if (config.NUQ_RABBITMQ_URL) {+ const connection = await amqp.connect(config.NUQ_RABBITMQ_URL);const channel = await connection.createChannel();await channel.assertQueue(this.queueName + ".prefetch", {durable: true,@@ -1038,7 +1039,7 @@ class NuQ<JobData = any, JobReturnValue = any> {}private readonly nuqWaitMode =- process.env.NUQ_WAIT_MODE === "listen" || process.env.NUQ_RABBITMQ_URL+ config.NUQ_WAIT_MODE === "listen" || config.NUQ_RABBITMQ_URL? ("listen" as const): ("poll" as const);@@ -1194,7 +1195,7 @@ class NuQ<JobData = any, JobReturnValue = any> {public async getJobToProcess(): Promise<NuQJob<any, any> | null> {const start = Date.now();try {- if (process.env.NUQ_RABBITMQ_URL) {+ if (config.NUQ_RABBITMQ_URL) {await this.startSender();if (this.sender) {@@ -1281,11 +1282,11 @@ class NuQ<JobData = any, JobReturnValue = any> {if (success) {const job = result.rows[0];- if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {+ if (this.nuqWaitMode === "listen" && !config.NUQ_RABBITMQ_URL) {await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [job.id + "|completed",]);- } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {+ } else if (config.NUQ_RABBITMQ_URL && job.listen_channel_id) {await this.sendJobEnd(job.id,"completed",@@ -1339,11 +1340,11 @@ class NuQ<JobData = any, JobReturnValue = any> {if (success) {const job = result.rows[0];- if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {+ if (this.nuqWaitMode === "listen" && !config.NUQ_RABBITMQ_URL) {await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [job.id + "|failed",]);- } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {+ } else if (config.NUQ_RABBITMQ_URL && job.listen_channel_id) {await this.sendJobEnd(job.id,"failed",diff --git a/apps/api/src/services/worker/redis.ts b/apps/api/src/services/worker/redis.tsindex bca1aa2a8..c98859152 100644--- a/apps/api/src/services/worker/redis.ts+++ b/apps/api/src/services/worker/redis.ts@@ -1,4 +1,5 @@import IORedis, { Redis } from "ioredis";+import { config } from "../../config";import { logger as _logger } from "../../lib/logger";const logger = _logger.child({ module: "nuq/redis" });@@ -57,7 +58,7 @@ type ScriptHashes = {const scripts: ScriptHashes = {} as ScriptHashes;-const redis = new IORedis(process.env.REDIS_URL!, {+const redis = new IORedis(config.REDIS_URL!, {lazyConnect: true,maxRetriesPerRequest: null,enableReadyCheck: false,diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 6c4d2deb7..0d665da87 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -1,4 +1,5 @@import { configDotenv } from "dotenv";+import { config } from "../../config";import * as Sentry from "@sentry/node";import http from "http";import https from "https";@@ -76,10 +77,8 @@ import { FeatureFlag } from "../../scraper/scrapeURL/engines";configDotenv();-const jobLockExtendInterval =- Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;-const jobLockExtensionTime =- Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;+const jobLockExtendInterval = config.JOB_LOCK_EXTEND_INTERVAL;+const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;if (require.main === module) {cacheableLookup.install(http.globalAgent);@@ -109,8 +108,8 @@ async function billScrapeJob();if (- job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&- process.env.USE_DB_AUTHENTICATION === "true"+ job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&+ config.USE_DB_AUTHENTICATION) {try {const billingJobId = uuidv7();@@ -1165,7 +1164,7 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {}try {- if (process.env.GCS_BUCKET_NAME && !job.data.skipNuq) {+ if (config.GCS_BUCKET_NAME && !job.data.skipNuq) {logger.debug("Job succeeded -- putting null in Redis");return null;} else {