Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

refactor(api): centralize firecrawl config (#2496)

tom166d067c8a6f
diff --git a/apps/api/src/__tests__/e2e_extract/index.test.ts b/apps/api/src/__tests__/e2e_extract/index.test.ts
index d1818643a..5547ff76c 100644
--- a/apps/api/src/__tests__/e2e_extract/index.test.ts
+++ b/apps/api/src/__tests__/e2e_extract/index.test.ts
@@ -1,12 +1,11 @@
import request from "supertest";
-import dotenv from "dotenv";
+import { config } from "../../config";
import {
FirecrawlCrawlResponse,
FirecrawlCrawlStatusResponse,
FirecrawlScrapeResponse,
} from "../../types";
-dotenv.config();
const TEST_URL = "http://127.0.0.1:3002";
describe("E2E Tests for Extract API Routes", () => {
@@ -15,7 +14,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/extract")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
urls: ["https://firecrawl.dev/*"],
@@ -53,7 +52,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/extract")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
urls: ["firecrawl.dev/*"],
@@ -91,7 +90,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/extract")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
urls: ["https://firecrawl.dev/*"],
@@ -128,7 +127,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/extract")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
urls: ["fivetran.com/*"],
@@ -153,7 +152,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/extract")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
urls: ["fivetran.com/*"],
@@ -184,7 +183,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/extract")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
urls: [
@@ -214,7 +213,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/extract")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
urls: ["https://mintlify.com/docs/*"],
@@ -258,7 +257,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/extract")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
urls: ["https://ericciarla.com/"],
@@ -291,7 +290,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/extract")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
urls: ["https://docs.firecrawl.dev"],
@@ -312,7 +311,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/extract")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
urls: [
@@ -348,7 +347,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.get("/v2/extract/None")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.send();
expect(response.statusCode).toBe(400);
@@ -364,7 +363,7 @@ describe("E2E Tests for Extract API Routes", () => {
async () => {
const response = await request(TEST_URL)
.get("/v2/extract/not-a-uuid")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.send();
expect(response.statusCode).toBe(400);
diff --git a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
index 532b1d222..3b9bd5151 100644
--- a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
@@ -1,20 +1,21 @@
import request from "supertest";
-import dotenv from "dotenv";
+import { config } from "../../config";
import { v7 as uuidv7 } from "uuid";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
-dotenv.config();
-
// const TEST_URL = 'http://localhost:3002'
const TEST_URL = "http://127.0.0.1:3002";
describe("E2E Tests for API Routes", () => {
+ let originalUseDbAuth: boolean | undefined;
+
beforeAll(() => {
- process.env.USE_DB_AUTHENTICATION = "true";
+ originalUseDbAuth = config.USE_DB_AUTHENTICATION;
+ config.USE_DB_AUTHENTICATION = true;
});
afterAll(() => {
- delete process.env.USE_DB_AUTHENTICATION;
+ config.USE_DB_AUTHENTICATION = originalUseDbAuth;
});
describe("GET /e2e-test", () => {
@@ -47,7 +48,7 @@ describe("E2E Tests for API Routes", () => {
const blocklistedUrl = "https://facebook.com/fake-test";
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: blocklistedUrl });
expect(response.statusCode).toBe(403);
@@ -59,7 +60,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://roastmywebsite.ai" });
expect(response.statusCode).toBe(200);
@@ -115,7 +116,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://roastmywebsite.ai",
@@ -141,7 +142,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://roastmywebsite.ai",
@@ -167,7 +168,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://arxiv.org/pdf/astro-ph/9301001.pdf" });
await new Promise(r => setTimeout(r, 6000));
@@ -190,7 +191,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://arxiv.org/pdf/astro-ph/9301001" });
await new Promise(r => setTimeout(r, 6000));
@@ -213,7 +214,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://arxiv.org/pdf/astro-ph/9301001.pdf",
@@ -237,7 +238,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const responseWithoutRemoveTags = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://www.scrapethissite.com/" });
expect(responseWithoutRemoveTags.statusCode).toBe(200);
@@ -261,7 +262,7 @@ describe("E2E Tests for API Routes", () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.scrapethissite.com/",
@@ -286,7 +287,7 @@ describe("E2E Tests for API Routes", () => {
// const startTime = Date.now();
// const response = await request(TEST_URL)
// .post("/v0/scrape")
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: "https://firecrawl.dev", pageOptions: { waitFor: 7000 } });
// const endTime = Date.now();
@@ -307,7 +308,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/400" });
await new Promise(r => setTimeout(r, 5000));
@@ -329,7 +330,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/401" });
await new Promise(r => setTimeout(r, 5000));
@@ -351,7 +352,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/403" });
@@ -373,7 +374,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/404" });
await new Promise(r => setTimeout(r, 5000));
@@ -395,7 +396,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/405" });
await new Promise(r => setTimeout(r, 5000));
@@ -417,7 +418,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/500" });
await new Promise(r => setTimeout(r, 5000));
@@ -457,7 +458,7 @@ describe("E2E Tests for API Routes", () => {
const blocklistedUrl = "https://twitter.com/fake-test";
const response = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: blocklistedUrl });
expect(response.statusCode).toBe(403);
@@ -469,7 +470,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
@@ -487,7 +488,7 @@ describe("E2E Tests for API Routes", () => {
// First request with the idempotency key
const firstResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.set("x-idempotency-key", uniqueIdempotencyKey)
.send({ url: "https://docs.firecrawl.dev" });
@@ -497,7 +498,7 @@ describe("E2E Tests for API Routes", () => {
// Second request with the same idempotency key
const secondResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.set("x-idempotency-key", uniqueIdempotencyKey)
.send({ url: "https://docs.firecrawl.dev" });
@@ -512,7 +513,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
@@ -528,7 +529,7 @@ describe("E2E Tests for API Routes", () => {
while (!isFinished) {
response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -572,7 +573,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
@@ -588,7 +589,7 @@ describe("E2E Tests for API Routes", () => {
while (!isFinished) {
response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -617,7 +618,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
@@ -630,7 +631,7 @@ describe("E2E Tests for API Routes", () => {
while (!isFinished) {
response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -667,7 +668,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.scrapethissite.com",
@@ -677,7 +678,7 @@ describe("E2E Tests for API Routes", () => {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
@@ -686,7 +687,7 @@ describe("E2E Tests for API Routes", () => {
while (!isCompleted) {
const statusCheckResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusCheckResponse.statusCode).toBe(200);
isCompleted = statusCheckResponse.body.status === "completed";
if (!isCompleted) {
@@ -695,7 +696,7 @@ describe("E2E Tests for API Routes", () => {
}
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
@@ -735,7 +736,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.scrapethissite.com/pages/",
@@ -745,7 +746,7 @@ describe("E2E Tests for API Routes", () => {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
@@ -754,7 +755,7 @@ describe("E2E Tests for API Routes", () => {
while (!isCompleted) {
const statusCheckResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusCheckResponse.statusCode).toBe(200);
isCompleted = statusCheckResponse.body.status === "completed";
if (!isCompleted) {
@@ -763,7 +764,7 @@ describe("E2E Tests for API Routes", () => {
}
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
@@ -797,7 +798,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.mendable.ai",
@@ -807,7 +808,7 @@ describe("E2E Tests for API Routes", () => {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
@@ -816,7 +817,7 @@ describe("E2E Tests for API Routes", () => {
while (!isCompleted) {
const statusCheckResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusCheckResponse.statusCode).toBe(200);
isCompleted = statusCheckResponse.body.status === "completed";
if (!isCompleted) {
@@ -825,7 +826,7 @@ describe("E2E Tests for API Routes", () => {
}
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
const testurls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL,
@@ -862,7 +863,7 @@ describe("E2E Tests for API Routes", () => {
// it.concurrent("should return a successful response with a valid API key and valid limit option", async () => {
// const crawlResponse = await request(TEST_URL)
// .post("/v0/crawl")
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({
// url: "https://mendable.ai",
@@ -871,7 +872,7 @@ describe("E2E Tests for API Routes", () => {
// const response = await request(TEST_URL)
// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
// expect(response.statusCode).toBe(200);
// expect(response.body).toHaveProperty("status");
// expect(response.body.status).toBe("active");
@@ -880,7 +881,7 @@ describe("E2E Tests for API Routes", () => {
// while (!isCompleted) {
// const statusCheckResponse = await request(TEST_URL)
// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
// expect(statusCheckResponse.statusCode).toBe(200);
// isCompleted = statusCheckResponse.body.status === "completed";
// if (!isCompleted) {
@@ -890,7 +891,7 @@ describe("E2E Tests for API Routes", () => {
// const completedResponse = await request(TEST_URL)
// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
// expect(completedResponse.statusCode).toBe(200);
// expect(completedResponse.body).toHaveProperty("status");
@@ -909,7 +910,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://roastmywebsite.ai",
@@ -919,7 +920,7 @@ describe("E2E Tests for API Routes", () => {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
@@ -928,7 +929,7 @@ describe("E2E Tests for API Routes", () => {
while (!isCompleted) {
const statusCheckResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusCheckResponse.statusCode).toBe(200);
isCompleted = statusCheckResponse.body.status === "completed";
if (!isCompleted) {
@@ -938,7 +939,7 @@ describe("E2E Tests for API Routes", () => {
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
@@ -976,7 +977,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlInitResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
@@ -996,7 +997,7 @@ describe("E2E Tests for API Routes", () => {
while (crawlStatus !== "completed") {
const statusResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlInitResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
crawlStatus = statusResponse.body.status;
if (statusResponse.body.data) {
crawlData = statusResponse.body.data;
@@ -1048,7 +1049,7 @@ describe("E2E Tests for API Routes", () => {
// const blocklistedUrl = "https://instagram.com/fake-test";
// const response = await request(TEST_URL)
// .post("/v0/crawlWebsitePreview")
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: blocklistedUrl });
// // is returning 429 instead of 403
@@ -1061,7 +1062,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev", timeout: 1000 });
@@ -1094,7 +1095,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/search")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ query: "test" });
expect(response.statusCode).toBe(200);
@@ -1127,7 +1128,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.get("/v0/crawl/status/invalidJobId")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(404);
},
);
@@ -1137,7 +1138,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://mendable.ai/blog" });
expect(crawlResponse.statusCode).toBe(200);
@@ -1148,7 +1149,7 @@ describe("E2E Tests for API Routes", () => {
while (!isCompleted) {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -1190,7 +1191,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://arxiv.org/pdf/astro-ph/9301001",
@@ -1214,7 +1215,7 @@ describe("E2E Tests for API Routes", () => {
while (!isCompleted) {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -1254,7 +1255,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://roastmywebsite.ai",
@@ -1264,7 +1265,7 @@ describe("E2E Tests for API Routes", () => {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
@@ -1275,7 +1276,7 @@ describe("E2E Tests for API Routes", () => {
while (!isFinished) {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -1314,7 +1315,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai/blog",
@@ -1329,7 +1330,7 @@ describe("E2E Tests for API Routes", () => {
while (!isFinished) {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -1374,7 +1375,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://jestjs.io" });
@@ -1384,7 +1385,7 @@ describe("E2E Tests for API Routes", () => {
const responseCancel = await request(TEST_URL)
.delete(`/v0/crawl/cancel/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(responseCancel.statusCode).toBe(200);
expect(responseCancel.body).toHaveProperty("status");
expect(responseCancel.body.status).toBe("cancelled");
@@ -1392,7 +1393,7 @@ describe("E2E Tests for API Routes", () => {
await new Promise(r => setTimeout(r, 10000));
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
@@ -1419,7 +1420,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
@@ -1472,7 +1473,7 @@ describe("E2E Tests for API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
@@ -1516,7 +1517,7 @@ describe("E2E Tests for API Routes", () => {
// it.concurrent("should extract data for the top 100 companies", async () => {
// const response = await request(TEST_URL)
// .post("/v0/scrape")
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({
// url: "https://companiesmarketcap.com/",
@@ -1578,7 +1579,7 @@ describe("E2E Tests for API Routes", () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://flutterbricks.com",
@@ -1596,7 +1597,7 @@ describe("E2E Tests for API Routes", () => {
while (!isFinished) {
statusResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusResponse.statusCode).toBe(200);
isFinished = statusResponse.body.status === "completed";
@@ -1634,7 +1635,7 @@ describe("E2E Tests for API Routes", () => {
// const crawlResponse = await request(TEST_URL)
// .post("/v0/crawl")
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({
// url: "https://flutterbricks.com",
@@ -1649,7 +1650,7 @@ describe("E2E Tests for API Routes", () => {
// while (!isFinished) {
// statusResponse = await request(TEST_URL)
// .get(`/v0/crawl/status/${jobId}`)
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
// expect(statusResponse.statusCode).toBe(200);
// isFinished = statusResponse.body.status === "completed";
@@ -1693,7 +1694,7 @@ describe("E2E Tests for API Routes", () => {
for (let i = 0; i < 5; i++) {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`)
+ .set("Authorization", `Bearer ${config.PREVIEW_TOKEN}`)
.set("Content-Type", "application/json")
.send({ url: "https://www.scrapethissite.com" });
@@ -1701,7 +1702,7 @@ describe("E2E Tests for API Routes", () => {
}
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`)
+ .set("Authorization", `Bearer ${config.PREVIEW_TOKEN}`)
.set("Content-Type", "application/json")
.send({ url: "https://www.scrapethissite.com" });
@@ -1712,10 +1713,10 @@ describe("E2E Tests for API Routes", () => {
});
// it.concurrent("should return 429 when rate limit is exceeded for API key", async () => {
- // for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) {
+ // for (let i = 0; i < parseInt(config.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) {
// const response = await request(TEST_URL)
// .post("/v0/scrape")
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: "https://www.scrapethissite.com" });
@@ -1724,7 +1725,7 @@ describe("E2E Tests for API Routes", () => {
// const response = await request(TEST_URL)
// .post("/v0/scrape")
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: "https://www.scrapethissite.com" });
@@ -1732,10 +1733,10 @@ describe("E2E Tests for API Routes", () => {
// }, 60000);
// it.concurrent("should return 429 when rate limit is exceeded for API key", async () => {
- // for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) {
+ // for (let i = 0; i < parseInt(config.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) {
// const response = await request(TEST_URL)
// .post("/v0/crawl")
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: "https://www.scrapethissite.com" });
@@ -1744,7 +1745,7 @@ describe("E2E Tests for API Routes", () => {
// const response = await request(TEST_URL)
// .post("/v0/crawl")
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: "https://www.scrapethissite.com" });
diff --git a/apps/api/src/__tests__/e2e_map/index.test.ts b/apps/api/src/__tests__/e2e_map/index.test.ts
index 30ec6776e..26f25f101 100644
--- a/apps/api/src/__tests__/e2e_map/index.test.ts
+++ b/apps/api/src/__tests__/e2e_map/index.test.ts
@@ -1,7 +1,6 @@
import request from "supertest";
-import dotenv from "dotenv";
+import { config } from "../../config";
-dotenv.config();
const TEST_URL = "http://127.0.0.1:3002";
describe("E2E Tests for Map API Routes", () => {
@@ -10,7 +9,7 @@ describe("E2E Tests for Map API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
@@ -32,7 +31,7 @@ describe("E2E Tests for Map API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
@@ -56,7 +55,7 @@ describe("E2E Tests for Map API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
@@ -79,7 +78,7 @@ describe("E2E Tests for Map API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
@@ -100,7 +99,7 @@ describe("E2E Tests for Map API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v1/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://geekflare.com/sitemap_index.xml",
diff --git a/apps/api/src/__tests__/e2e_map/v2_map.test.ts b/apps/api/src/__tests__/e2e_map/v2_map.test.ts
index 7d3987a80..ec255fa9d 100644
--- a/apps/api/src/__tests__/e2e_map/v2_map.test.ts
+++ b/apps/api/src/__tests__/e2e_map/v2_map.test.ts
@@ -1,7 +1,6 @@
import request from "supertest";
-import dotenv from "dotenv";
+import { config } from "../../config";
-dotenv.config();
const TEST_URL = "http://127.0.0.1:3002";
describe("E2E Tests for Map V2 API Routes", () => {
@@ -10,7 +9,7 @@ describe("E2E Tests for Map V2 API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v2/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://roastmywebsite.ai",
@@ -46,7 +45,7 @@ describe("E2E Tests for Map V2 API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v2/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
@@ -67,7 +66,7 @@ describe("E2E Tests for Map V2 API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v2/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
@@ -87,7 +86,7 @@ describe("E2E Tests for Map V2 API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v2/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
@@ -109,7 +108,7 @@ describe("E2E Tests for Map V2 API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v2/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
diff --git a/apps/api/src/__tests__/e2e_noAuth/index.test.ts b/apps/api/src/__tests__/e2e_noAuth/index.test.ts
index 396521e0b..428202a85 100644
--- a/apps/api/src/__tests__/e2e_noAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_noAuth/index.test.ts
@@ -1,33 +1,41 @@
import request from "supertest";
-import dotenv from "dotenv";
+import { config } from "../../config";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
const fs = require("fs");
const path = require("path");
-dotenv.config();
-
const TEST_URL = "http://127.0.0.1:3002";
describe("E2E Tests for API Routes with No Authentication", () => {
- let originalEnv: NodeJS.ProcessEnv;
+ let originalConfig: Partial<typeof config>;
- // save original process.env
+ // save original config values
beforeAll(() => {
- originalEnv = { ...process.env };
- process.env.USE_DB_AUTHENTICATION = "false";
- process.env.SUPABASE_ANON_TOKEN = "";
- process.env.SUPABASE_URL = "";
- process.env.SUPABASE_SERVICE_TOKEN = "";
- process.env.OPENAI_API_KEY = "";
- process.env.BULL_AUTH_KEY = "";
- process.env.PLAYWRIGHT_MICROSERVICE_URL = "";
- process.env.LLAMAPARSE_API_KEY = "";
- process.env.TEST_API_KEY = "";
+ originalConfig = {
+ USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,
+ SUPABASE_ANON_TOKEN: config.SUPABASE_ANON_TOKEN,
+ SUPABASE_URL: config.SUPABASE_URL,
+ SUPABASE_SERVICE_TOKEN: config.SUPABASE_SERVICE_TOKEN,
+ OPENAI_API_KEY: config.OPENAI_API_KEY,
+ BULL_AUTH_KEY: config.BULL_AUTH_KEY,
+ PLAYWRIGHT_MICROSERVICE_URL: config.PLAYWRIGHT_MICROSERVICE_URL,
+ LLAMAPARSE_API_KEY: config.LLAMAPARSE_API_KEY,
+ TEST_API_KEY: config.TEST_API_KEY,
+ };
+ config.USE_DB_AUTHENTICATION = false;
+ config.SUPABASE_ANON_TOKEN = "";
+ config.SUPABASE_URL = "";
+ config.SUPABASE_SERVICE_TOKEN = "";
+ config.OPENAI_API_KEY = "";
+ config.BULL_AUTH_KEY = "";
+ config.PLAYWRIGHT_MICROSERVICE_URL = "";
+ config.LLAMAPARSE_API_KEY = "";
+ config.TEST_API_KEY = "";
});
- // restore original process.env
+ // restore original config values
afterAll(() => {
- process.env = originalEnv;
+ Object.assign(config, originalConfig);
});
describe("GET /e2e-test", () => {
diff --git a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
index b609050d8..ffb3eab49 100644
--- a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
@@ -1,4 +1,5 @@
import request from "supertest";
+import { config } from "../../config";
import { configDotenv } from "dotenv";
import { ScrapeRequestInput } from "../../controllers/v1/types";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
@@ -8,23 +9,20 @@ const TEST_URL = "http://127.0.0.1:3002";
describe("E2E Tests for v1 API Routes", () => {
beforeAll(() => {
- process.env.USE_DB_AUTHENTICATION = "true";
+ config.USE_DB_AUTHENTICATION = true;
});
afterAll(() => {
- delete process.env.USE_DB_AUTHENTICATION;
+ delete config.USE_DB_AUTHENTICATION;
});
describe("GET /is-production", () => {
it.concurrent("should return the production status", async () => {
const response: any = await request(TEST_URL).get("/is-production");
- console.log(
- "process.env.USE_DB_AUTHENTICATION",
- process.env.USE_DB_AUTHENTICATION,
- );
- console.log("?", process.env.USE_DB_AUTHENTICATION === "true");
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
+ console.log("config.USE_DB_AUTHENTICATION", config.USE_DB_AUTHENTICATION);
+ console.log("?", config.USE_DB_AUTHENTICATION);
+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;
console.log("!!useDbAuthentication", !!useDbAuthentication);
console.log("!useDbAuthentication", !useDbAuthentication);
@@ -49,7 +47,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -78,7 +76,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -130,7 +128,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -187,7 +185,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -215,7 +213,7 @@ describe("E2E Tests for v1 API Routes", () => {
};
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
await new Promise(r => setTimeout(r, 6000));
@@ -243,7 +241,7 @@ describe("E2E Tests for v1 API Routes", () => {
};
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
await new Promise(r => setTimeout(r, 6000));
@@ -273,7 +271,7 @@ describe("E2E Tests for v1 API Routes", () => {
};
const responseWithoutRemoveTags: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
expect(responseWithoutRemoveTags.statusCode).toBe(200);
@@ -299,7 +297,7 @@ describe("E2E Tests for v1 API Routes", () => {
};
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequestWithRemoveTags);
@@ -322,7 +320,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/400" });
await new Promise(r => setTimeout(r, 5000));
@@ -344,7 +342,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/401" });
await new Promise(r => setTimeout(r, 5000));
@@ -365,7 +363,7 @@ describe("E2E Tests for v1 API Routes", () => {
// it.concurrent('should return a successful response for a scrape with 403 page', async () => {
// const response: any = await request(TEST_URL)
// .post('/v1/scrape')
- // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
+ // .set('Authorization', `Bearer ${config.TEST_API_KEY}`)
// .set('Content-Type', 'application/json')
// .send({ url: 'https://httpstat.us/403' });
// await new Promise((r) => setTimeout(r, 5000));
@@ -385,7 +383,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/404" });
await new Promise(r => setTimeout(r, 5000));
@@ -405,7 +403,7 @@ describe("E2E Tests for v1 API Routes", () => {
// it.concurrent('should return a successful response for a scrape with 405 page', async () => {
// const response: any = await request(TEST_URL)
// .post('/v1/scrape')
- // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
+ // .set('Authorization', `Bearer ${config.TEST_API_KEY}`)
// .set('Content-Type', 'application/json')
// .send({ url: 'https://httpstat.us/405' });
// await new Promise((r) => setTimeout(r, 5000));
@@ -423,7 +421,7 @@ describe("E2E Tests for v1 API Routes", () => {
// it.concurrent('should return a successful response for a scrape with 500 page', async () => {
// const response: any = await request(TEST_URL)
// .post('/v1/scrape')
- // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
+ // .set('Authorization', `Bearer ${config.TEST_API_KEY}`)
// .set('Content-Type', 'application/json')
// .send({ url: 'https://httpstat.us/500' });
// await new Promise((r) => setTimeout(r, 5000));
@@ -443,7 +441,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev", timeout: 1000 });
@@ -462,7 +460,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -494,7 +492,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -525,7 +523,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -575,7 +573,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(mapRequest);
@@ -601,7 +599,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(mapRequest);
@@ -629,7 +627,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(mapRequest);
@@ -661,7 +659,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(mapRequest);
@@ -694,7 +692,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(mapRequest);
@@ -720,7 +718,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v1/map")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(mapRequest);
@@ -745,7 +743,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response = await request(TEST_URL)
.post("/v1/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -768,7 +766,7 @@ describe("E2E Tests for v1 API Routes", () => {
it.concurrent("should return a successful response", async () => {
const response = await request(TEST_URL)
.post("/v1/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
@@ -787,7 +785,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v1/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
@@ -801,7 +799,7 @@ describe("E2E Tests for v1 API Routes", () => {
while (!isFinished) {
response = await request(TEST_URL)
.get(`/v1/crawl/${crawlResponse.body.id}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -815,7 +813,7 @@ describe("E2E Tests for v1 API Routes", () => {
await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the database
const completedResponse = await request(TEST_URL)
.get(`/v1/crawl/${crawlResponse.body.id}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
const urls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL,
@@ -843,7 +841,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v1/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl.dev",
@@ -857,7 +855,7 @@ describe("E2E Tests for v1 API Routes", () => {
while (!isFinished) {
response = await request(TEST_URL)
.get(`/v1/crawl/${crawlResponse.body.id}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -871,7 +869,7 @@ describe("E2E Tests for v1 API Routes", () => {
await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the database
const completedResponse = await request(TEST_URL)
.get(`/v1/crawl/${crawlResponse.body.id}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
const urls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL,
@@ -889,7 +887,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v1/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.scrapethissite.com",
@@ -899,7 +897,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response = await request(TEST_URL)
.get(`/v1/crawl/${crawlResponse.body.id}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting", "completed", "scraping"]).toContain(
@@ -910,7 +908,7 @@ describe("E2E Tests for v1 API Routes", () => {
while (!isCompleted) {
const statusCheckResponse = await request(TEST_URL)
.get(`/v1/crawl/${crawlResponse.body.id}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusCheckResponse.statusCode).toBe(200);
isCompleted = statusCheckResponse.body.status === "completed";
if (!isCompleted) {
@@ -919,7 +917,7 @@ describe("E2E Tests for v1 API Routes", () => {
}
const completedResponse = await request(TEST_URL)
.get(`/v1/crawl/${crawlResponse.body.id}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
@@ -972,7 +970,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const response = await request(TEST_URL)
.get("/v1/crawl/invalidJobId")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(404);
},
);
@@ -982,7 +980,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v1/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://docs.firecrawl.dev" });
expect(crawlResponse.statusCode).toBe(200);
@@ -992,7 +990,7 @@ describe("E2E Tests for v1 API Routes", () => {
while (!isCompleted) {
const response = await request(TEST_URL)
.get(`/v1/crawl/${crawlResponse.body.id}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -1006,7 +1004,7 @@ describe("E2E Tests for v1 API Routes", () => {
await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the database
const completedResponse = await request(TEST_URL)
.get(`/v1/crawl/${crawlResponse.body.id}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
@@ -1031,7 +1029,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v1/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://docs.firecrawl.dev", limit: 10 });
@@ -1041,7 +1039,7 @@ describe("E2E Tests for v1 API Routes", () => {
const responseCancel = await request(TEST_URL)
.delete(`/v1/crawl/${crawlResponse.body.id}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(responseCancel.statusCode).toBe(200);
expect(responseCancel.body).toHaveProperty("status");
expect(responseCancel.body.status).toBe("cancelled");
@@ -1049,7 +1047,7 @@ describe("E2E Tests for v1 API Routes", () => {
await new Promise(r => setTimeout(r, 10000));
const completedResponse = await request(TEST_URL)
.get(`/v1/crawl/${crawlResponse.body.id}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
diff --git a/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts
index 5eb39f27a..268ece80b 100644
--- a/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts
+++ b/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts
@@ -1,4 +1,5 @@
import request from "supertest";
+import { config } from "../../config";
import { configDotenv } from "dotenv";
import { ScrapeRequest } from "../../controllers/v1/types";
@@ -12,7 +13,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/403" });
@@ -37,7 +38,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -82,7 +83,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -115,7 +116,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -151,7 +152,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -178,7 +179,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -208,7 +209,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -239,7 +240,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -269,7 +270,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -294,7 +295,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -315,7 +316,7 @@ describe("E2E Tests for v1 API Routes", () => {
async () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://arxiv.org/pdf/astro-ph/9301001.pdf" });
await new Promise(r => setTimeout(r, 6000));
@@ -335,7 +336,7 @@ describe("E2E Tests for v1 API Routes", () => {
const responseNoParsePDF: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://arxiv.org/pdf/astro-ph/9301001.pdf",
@@ -367,7 +368,7 @@ describe("E2E Tests for v1 API Routes", () => {
// const response: any = await request(FIRECRAWL_API_URL)
// .post("/v1/scrape")
- // .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ // .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send(scrapeRequest);
@@ -386,7 +387,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
console.log("Error1a");
@@ -408,7 +409,7 @@ describe("E2E Tests for v1 API Routes", () => {
FIRECRAWL_API_URL,
)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequestWithSkipTlsVerification);
@@ -436,7 +437,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -466,7 +467,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -497,7 +498,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -538,7 +539,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -586,7 +587,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -622,7 +623,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -655,7 +656,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
@@ -692,7 +693,7 @@ describe("E2E Tests for v1 API Routes", () => {
const response: any = await request(FIRECRAWL_API_URL)
.post("/v1/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send(scrapeRequest);
diff --git a/apps/api/src/__tests__/e2e_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
index e5a300092..8cb7e53df 100644
--- a/apps/api/src/__tests__/e2e_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_withAuth/index.test.ts
@@ -1,16 +1,15 @@
import request from "supertest";
-import dotenv from "dotenv";
+import { config } from "../../config";
-dotenv.config();
const TEST_URL = "http://127.0.0.1:3002";
describe("E2E Tests for v0 API Routes", () => {
beforeAll(() => {
- process.env.USE_DB_AUTHENTICATION = "true";
+ config.USE_DB_AUTHENTICATION = true;
});
afterAll(() => {
- delete process.env.USE_DB_AUTHENTICATION;
+ delete config.USE_DB_AUTHENTICATION;
});
describe("GET /is-production", () => {
@@ -44,7 +43,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://roastmywebsite.ai" });
expect(response.statusCode).toBe(200);
@@ -88,7 +87,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://roastmywebsite.ai",
@@ -114,7 +113,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://arxiv.org/pdf/astro-ph/9301001.pdf" });
await new Promise(r => setTimeout(r, 6000));
@@ -137,7 +136,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://arxiv.org/pdf/astro-ph/9301001" });
await new Promise(r => setTimeout(r, 6000));
@@ -160,7 +159,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const responseWithoutRemoveTags: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://www.scrapethissite.com/" });
expect(responseWithoutRemoveTags.statusCode).toBe(200);
@@ -184,7 +183,7 @@ describe("E2E Tests for v0 API Routes", () => {
const response: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.scrapethissite.com/",
@@ -209,7 +208,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/400" });
await new Promise(r => setTimeout(r, 5000));
@@ -231,7 +230,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/401" });
await new Promise(r => setTimeout(r, 5000));
@@ -253,7 +252,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/403" });
@@ -275,7 +274,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/404" });
await new Promise(r => setTimeout(r, 5000));
@@ -294,7 +293,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/405" });
await new Promise(r => setTimeout(r, 5000));
@@ -313,7 +312,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/500" });
await new Promise(r => setTimeout(r, 5000));
@@ -351,7 +350,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
@@ -367,7 +366,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const crawlResponse: any = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
@@ -383,7 +382,7 @@ describe("E2E Tests for v0 API Routes", () => {
while (!isFinished) {
response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -397,7 +396,7 @@ describe("E2E Tests for v0 API Routes", () => {
await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the database
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
const urls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL,
@@ -430,7 +429,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const crawlResponse: any = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
@@ -446,7 +445,7 @@ describe("E2E Tests for v0 API Routes", () => {
while (!isFinished) {
response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -460,7 +459,7 @@ describe("E2E Tests for v0 API Routes", () => {
await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the database
const completedResponse: any = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
const urls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL,
@@ -478,7 +477,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const crawlResponse: any = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.scrapethissite.com",
@@ -488,7 +487,7 @@ describe("E2E Tests for v0 API Routes", () => {
const response: any = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
@@ -497,7 +496,7 @@ describe("E2E Tests for v0 API Routes", () => {
while (!isCompleted) {
const statusCheckResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusCheckResponse.statusCode).toBe(200);
isCompleted = statusCheckResponse.body.status === "completed";
if (!isCompleted) {
@@ -506,7 +505,7 @@ describe("E2E Tests for v0 API Routes", () => {
}
const completedResponse: any = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
@@ -567,7 +566,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response: any = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev", timeout: 1000 });
@@ -600,7 +599,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/search")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ query: "test" });
expect(response.statusCode).toBe(200);
@@ -633,7 +632,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response = await request(TEST_URL)
.get("/v0/crawl/status/invalidJobId")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(404);
},
);
@@ -643,7 +642,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev/blog" });
expect(crawlResponse.statusCode).toBe(200);
@@ -653,7 +652,7 @@ describe("E2E Tests for v0 API Routes", () => {
while (!isCompleted) {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
@@ -667,7 +666,7 @@ describe("E2E Tests for v0 API Routes", () => {
await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the database
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
@@ -699,7 +698,7 @@ describe("E2E Tests for v0 API Routes", () => {
// it.concurrent('should return a successful response for a valid crawl job with PDF files without explicit .pdf extension ', async () => {
// const crawlResponse = await request(TEST_URL)
// .post('/v0/crawl')
- // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
+ // .set('Authorization', `Bearer ${config.TEST_API_KEY}`)
// .set('Content-Type', 'application/json')
// .send({ url: 'https://arxiv.org/list/astro-ph/1993-01',
// crawlerOptions: {
@@ -714,7 +713,7 @@ describe("E2E Tests for v0 API Routes", () => {
// while (!isCompleted) {
// const response = await request(TEST_URL)
// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- // .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`);
+ // .set('Authorization', `Bearer ${config.TEST_API_KEY}`);
// expect(response.statusCode).toBe(200);
// expect(response.body).toHaveProperty('status');
@@ -746,7 +745,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://docs.tatum.io",
@@ -759,7 +758,7 @@ describe("E2E Tests for v0 API Routes", () => {
const responseCancel = await request(TEST_URL)
.delete(`/v0/crawl/cancel/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(responseCancel.statusCode).toBe(200);
expect(responseCancel.body).toHaveProperty("status");
expect(responseCancel.body.status).toBe("cancelled");
@@ -767,7 +766,7 @@ describe("E2E Tests for v0 API Routes", () => {
await new Promise(r => setTimeout(r, 10000));
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.maxResponseSize(4000000000);
expect(completedResponse.statusCode).toBe(200);
@@ -811,7 +810,7 @@ describe("E2E Tests for v0 API Routes", () => {
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index 11b6f9632..a802fd8f2 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -1,4 +1,5 @@
import { configDotenv } from "dotenv";
+import { config } from "../../config";
configDotenv();
import { TeamFlags } from "../../controllers/v1/types";
@@ -7,7 +8,7 @@ import { TeamFlags } from "../../controllers/v1/types";
// Configuration
// =========================================
-export const TEST_API_URL = process.env.TEST_API_URL || "http://127.0.0.1:3002";
+export const TEST_API_URL = config.TEST_API_URL;
export const TEST_URL = TEST_API_URL; // backwards compat temp
const stripTrailingSlash = (url: string) => {
@@ -15,21 +16,17 @@ const stripTrailingSlash = (url: string) => {
return url.endsWith("/") ? url.substring(0, url.length - 1) : url;
};
-export const TEST_SUITE_WEBSITE = stripTrailingSlash(
- process.env.TEST_SUITE_WEBSITE || "http://127.0.0.1:4321",
-);
+export const TEST_SUITE_WEBSITE = stripTrailingSlash(config.TEST_SUITE_WEBSITE);
-export const TEST_SELF_HOST = process.env.TEST_SUITE_SELF_HOSTED === "true";
+export const TEST_SELF_HOST = !!config.TEST_SUITE_SELF_HOSTED;
export const TEST_PRODUCTION = !TEST_SELF_HOST;
// TODO: do we want to run AI tests when users run this command locally? It may lead to increased spending for them, depending on configuration
-export const HAS_AI = !!(
- process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL
-);
-export const HAS_PLAYWRIGHT = !!process.env.PLAYWRIGHT_MICROSERVICE_URL;
-export const HAS_PROXY = !!process.env.PROXY_SERVER;
+export const HAS_AI = !!(config.OPENAI_API_KEY || config.OLLAMA_BASE_URL);
+export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;
+export const HAS_PROXY = !!config.PROXY_SERVER;
-export const HAS_SEARCH = TEST_PRODUCTION || !!process.env.SEARXNG_ENDPOINT;
+export const HAS_SEARCH = TEST_PRODUCTION || !!config.SEARXNG_ENDPOINT;
const isLocalUrl = (x: string) =>
/^https?:\/\/(localhost|127\.0\.0\.1|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})(:\d+)?([\/?#]|$)/i.test(
@@ -52,7 +49,7 @@ export const createTestIdUrl = () =>
if (isLocalUrl(TEST_SUITE_WEBSITE)) {
if (TEST_SELF_HOST) {
- process.env.ALLOW_LOCAL_WEBHOOKS = "true";
+ config.ALLOW_LOCAL_WEBHOOKS = true;
} else {
throw new Error(
"TEST_SUITE_WEBSITE cannot be a local address while testing in production",
@@ -79,25 +76,25 @@ export type IdmuxRequest = {
};
export async function idmux(req: IdmuxRequest): Promise<Identity> {
- if (!process.env.IDMUX_URL) {
+ if (!config.IDMUX_URL) {
if (TEST_PRODUCTION) {
console.warn("IDMUX_URL is not set, using test API key and team ID");
}
return {
- apiKey: process.env.TEST_API_KEY!,
- teamId: process.env.TEST_TEAM_ID!,
+ apiKey: config.TEST_API_KEY!,
+ teamId: config.TEST_TEAM_ID!,
};
}
- let runNumber = parseInt(process.env.GITHUB_RUN_NUMBER!);
+ let runNumber = parseInt(config.GITHUB_RUN_NUMBER!);
if (isNaN(runNumber) || runNumber === null || runNumber === undefined) {
runNumber = 0;
}
- const res = await fetch(process.env.IDMUX_URL + "/", {
+ const res = await fetch(config.IDMUX_URL + "/", {
method: "POST",
body: JSON.stringify({
- refName: process.env.GITHUB_REF_NAME!,
+ refName: config.GITHUB_REF_NAME!,
runNumber,
concurrency: req.concurrency ?? 100,
...req,
diff --git a/apps/api/src/__tests__/snips/v1/lib.ts b/apps/api/src/__tests__/snips/v1/lib.ts
index 5664cb87f..9c905c466 100644
--- a/apps/api/src/__tests__/snips/v1/lib.ts
+++ b/apps/api/src/__tests__/snips/v1/lib.ts
@@ -1,3 +1,4 @@
+import { config } from "../../../config";
import {
ScrapeRequestInput,
Document,
@@ -548,7 +549,7 @@ export async function batchScrapeWithConcurrencyTracking(
export async function zdrcleaner(teamId: string) {
const res = await request(TEST_API_URL)
- .get(`/admin/${process.env.BULL_AUTH_KEY}/zdrcleaner`)
+ .get(`/admin/${config.BULL_AUTH_KEY}/zdrcleaner`)
.query({ teamId });
expect(res.statusCode).toBe(200);
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.ts
index 99b30825f..31ded4a57 100644
--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts
@@ -1,3 +1,4 @@
+import { config } from "../../../config";
import {
createTestIdUrl,
describeIf,
@@ -188,7 +189,7 @@ describe("Scrape tests", () => {
);
expect(response.markdown?.trim()).toContain(
- process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],
+ config.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],
);
},
scrapeTimeout,
@@ -208,7 +209,7 @@ describe("Scrape tests", () => {
);
expect(response.markdown?.trim()).toContain(
- process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],
+ config.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],
);
},
scrapeTimeout,
diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.ts
index 7dc175164..2a119a4c8 100644
--- a/apps/api/src/__tests__/snips/v2/lib.ts
+++ b/apps/api/src/__tests__/snips/v2/lib.ts
@@ -1,3 +1,4 @@
+import { config } from "../../../config";
import {
ScrapeRequestInput,
Document,
@@ -547,7 +548,7 @@ export async function extractRaw(
export async function zdrcleaner(teamId: string) {
const res = await request(TEST_API_URL)
- .get(`/admin/${process.env.BULL_AUTH_KEY}/zdrcleaner`)
+ .get(`/admin/${config.BULL_AUTH_KEY}/zdrcleaner`)
.query({ teamId });
expect(res.statusCode).toBe(200);
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index 676b7f735..9719c92bf 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -1,3 +1,4 @@
+import { config } from "../../../config";
import {
createTestIdUrl,
describeIf,
@@ -236,7 +237,7 @@ describe("Scrape tests", () => {
);
expect(response.markdown?.trim()).toContain(
- process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],
+ config.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],
);
},
scrapeTimeout,
@@ -255,7 +256,7 @@ describe("Scrape tests", () => {
);
expect(response.markdown?.trim()).toContain(
- process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],
+ config.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0],
);
},
scrapeTimeout,
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
new file mode 100644
index 000000000..1d8c28893
--- /dev/null
+++ b/apps/api/src/config.ts
@@ -0,0 +1,192 @@
+import "dotenv/config";
+import { z } from "zod";
+
+/* Codecs */
+const delimitedList = (separator = ",") => {
+ return z.codec(z.string(), z.array(z.string()), {
+ decode: str => (str ? str.split(separator).map(s => s.trim()) : []),
+ encode: arr => arr.join(separator),
+ });
+};
+
+/* Schema */
+const configSchema = z.object({
+ // Application
+ ENV: z.string().optional(),
+ HOST: z.string().default("localhost"),
+ PORT: z.coerce.number().default(3002),
+ IS_PRODUCTION: z.stringbool().optional(),
+ IS_KUBERNETES: z.stringbool().optional(),
+ FIRECRAWL_APP_HOST: z.string().default("firecrawl-app-service"),
+ FIRECRAWL_APP_PORT: z.string().default("3002"),
+ FIRECRAWL_APP_SCHEME: z.string().default("http"),
+ LOGGING_LEVEL: z.string().optional(),
+
+ // Express
+ EXPRESS_TRUST_PROXY: z.coerce.number().optional(),
+
+ // API Keys & Authentication
+ BULL_AUTH_KEY: z.string().optional(),
+ OPENAI_API_KEY: z.string().optional(),
+ OPENAI_BASE_URL: z.string().optional(),
+ OPENROUTER_API_KEY: z.string().optional(),
+ LLAMAPARSE_API_KEY: z.string().optional(),
+ STRIPE_SECRET_KEY: z.string().optional(),
+ RESEND_API_KEY: z.string().optional(),
+ PREVIEW_TOKEN: z.string().optional(),
+ SEARCH_PREVIEW_TOKEN: z.string().optional(),
+ SEARCH_SERVICE_API_SECRET: z.string().optional(),
+
+ // Database & Storage
+ POSTGRES_HOST: z.string().default("localhost"),
+ POSTGRES_PORT: z.string().default("5432"),
+ POSTGRES_DB: z.string().default("postgres"),
+ POSTGRES_USER: z.string().default("postgres"),
+ POSTGRES_PASSWORD: z.string().default("postgres"),
+ REDIS_URL: z.string().optional(),
+ REDIS_EVICT_URL: z.string().optional(),
+ REDIS_RATE_LIMIT_URL: z.string().optional(),
+ NUQ_DATABASE_URL: z.string().optional(),
+ NUQ_DATABASE_URL_LISTEN: z.string().optional(),
+ NUQ_RABBITMQ_URL: z.string().optional(),
+
+ // Supabase
+ SUPABASE_URL: z.string().optional(),
+ SUPABASE_ANON_TOKEN: z.string().optional(),
+ SUPABASE_SERVICE_TOKEN: z.string().optional(),
+ SUPABASE_REPLICA_URL: z.string().optional(),
+ SUPABASE_ACUC_URL: z.string().optional(),
+ SUPABASE_ACUC_SERVICE_TOKEN: z.string().optional(),
+ INDEX_SUPABASE_URL: z.string().optional(),
+ INDEX_SUPABASE_SERVICE_TOKEN: z.string().optional(),
+ SEARCH_INDEX_SUPABASE_URL: z.string().optional(),
+
+ // Google Cloud Storage
+ GCS_BUCKET_NAME: z.string().optional(),
+ GCS_CREDENTIALS: z.string().optional(),
+ GCS_FIRE_ENGINE_BUCKET_NAME: z.string().optional(),
+ GCS_INDEX_BUCKET_NAME: z.string().optional(),
+ GCS_MEDIA_BUCKET_NAME: z.string().optional(),
+
+ // Fire Engine
+ FIRE_ENGINE_BETA_URL: z.string().optional(),
+ FIRE_ENGINE_STAGING_URL: z.string().optional(),
+ FIRE_ENGINE_AB_HOST: z.string().optional(),
+ FIRE_ENGINE_AB_RATE: z.coerce.number().optional(),
+
+ // ScrapeURL
+ SCRAPEURL_AB_HOST: z.string().optional(),
+ SCRAPEURL_AB_RATE: z.coerce.number().optional(),
+ SCRAPEURL_AB_EXTEND_MAXAGE: z.stringbool().optional(),
+ SCRAPEURL_ENGINE_WATERFALL_DELAY_MS: z.coerce.number().default(0),
+
+ // Search Services
+ SEARXNG_ENDPOINT: z.string().optional(),
+ SEARXNG_ENGINES: z.string().optional(),
+ SEARXNG_CATEGORIES: z.string().optional(),
+ SEARCH_SERVICE_URL: z.string().optional(),
+ SEARCH_INDEX_SAMPLE_RATE: z.coerce.number().default(0.1),
+ ENABLE_SEARCH_INDEX: z.stringbool().optional(),
+
+ // Worker Configuration
+ WORKER_PORT: z.coerce.number().default(3005),
+ NUQ_WORKER_PORT: z.coerce.number().default(3000),
+ NUQ_WORKER_START_PORT: z.coerce.number().default(3006),
+ NUQ_WORKER_COUNT: z.coerce.number().default(5),
+ NUQ_PREFETCH_WORKER_PORT: z.coerce.number().default(3011),
+ EXTRACT_WORKER_PORT: z.coerce.number().default(3004),
+ NUQ_WAIT_MODE: z.string().optional(),
+
+ // Job & Lock Management
+ JOB_LOCK_EXTEND_INTERVAL: z.coerce.number().default(10000),
+ JOB_LOCK_EXTENSION_TIME: z.coerce.number().default(60000),
+ WORKER_LOCK_DURATION: z.coerce.number().default(60000),
+ WORKER_STALLED_CHECK_INTERVAL: z.coerce.number().default(30000),
+ CONNECTION_MONITOR_INTERVAL: z.coerce.number().default(10),
+ CANT_ACCEPT_CONNECTION_INTERVAL: z.coerce.number().default(2000),
+
+ // Proxy
+ PROXY_SERVER: z.string().optional(),
+ PROXY_USERNAME: z.string().optional(),
+ PROXY_PASSWORD: z.string().optional(),
+
+ // External Services
+ PLAYWRIGHT_MICROSERVICE_URL: z.string().optional(),
+ HTML_TO_MARKDOWN_SERVICE_URL: z.string().default("http://localhost:8080"),
+ SMART_SCRAPE_API_URL: z.string().optional(),
+
+ // PDF Processing
+ PDF_MU_V2_BASE_URL: z.string().optional(),
+ PDF_MU_V2_API_KEY: z.string().optional(),
+ PDF_MU_V2_EXPERIMENT: z.string().optional(),
+ PDF_MU_V2_EXPERIMENT_PERCENT: z.coerce.number().default(100),
+
+ // RunPod
+ RUNPOD_MU_API_KEY: z.string().optional(),
+ RUNPOD_MU_POD_ID: z.string().optional(),
+
+ // Webhooks
+ SELF_HOSTED_WEBHOOK_URL: z.string().optional(),
+ SELF_HOSTED_WEBHOOK_HMAC_SECRET: z.string().optional(),
+ SLACK_WEBHOOK_URL: z.string().optional(),
+ SLACK_ADMIN_WEBHOOK_URL: z.string().optional(),
+ DISABLE_WEBHOOK_DELIVERY: z.stringbool().optional(),
+ ALLOW_LOCAL_WEBHOOKS: z.stringbool().optional(),
+
+ // Firecrawl Features
+ FIRECRAWL_DEBUG_FILTER_LINKS: z.stringbool().optional(),
+ FIRECRAWL_LOG_TO_FILE: z.stringbool().optional(),
+ FIRECRAWL_SAVE_MOCKS: z.stringbool().optional(),
+ FIRECRAWL_INDEX_WRITE_ONLY: z.stringbool().optional(),
+ DISABLE_BLOCKLIST: z.stringbool().optional(),
+ FORCED_ENGINE_DOMAINS: z.string().optional(),
+ DEBUG_BRANDING: z.stringbool().optional(),
+
+ // AI/ML
+ MODEL_NAME: z.string().default("gpt-4o-mini"),
+ MODEL_EMBEDDING_NAME: z.string().optional(),
+ OLLAMA_BASE_URL: z.string().optional(),
+ VERTEX_CREDENTIALS: z.string().optional(),
+
+ // Rate Limiting
+ RATE_LIMIT_TEST_API_KEY_SCRAPE: z.coerce.number().optional(),
+ RATE_LIMIT_TEST_API_KEY_CRAWL: z.coerce.number().optional(),
+
+ // Testing
+ TEST_API_KEY: z.string().optional(),
+ TEST_API_URL: z.string().default("http://127.0.0.1:3002"),
+ TEST_TEAM_ID: z.string().optional(),
+ TEST_SUITE_SELF_HOSTED: z.stringbool().optional(),
+ TEST_SUITE_WEBSITE: z.string().default("http://127.0.0.1:4321"),
+ USE_DB_AUTHENTICATION: z.stringbool().optional(),
+
+ // Indexing
+ BACKGROUND_INDEX_TEAM_ID: z.string().optional(),
+ PRECRAWL_TEAM_ID: z.string().optional(),
+
+ // Payment (x402)
+ X402_ENDPOINT_PRICE_USD: z.string().optional(),
+ X402_NETWORK: z.string().optional(),
+ X402_PAY_TO_ADDRESS: z.string().optional(),
+
+ // System
+ MAX_CPU: z.coerce.number().default(0.8),
+ MAX_RAM: z.coerce.number().default(0.8),
+ SYS_INFO_MAX_CACHE_DURATION: z.coerce.number().default(150),
+ USE_GO_MARKDOWN_PARSER: z.stringbool().optional(),
+
+ // Sentry
+ SENTRY_DSN: z.string().optional(),
+ SENTRY_TRACE_SAMPLE_RATE: z.coerce.number().default(0.01),
+ SENTRY_ERROR_SAMPLE_RATE: z.coerce.number().default(0.05),
+ SENTRY_ENVIRONMENT: z.string().default("production"),
+ NUQ_POD_NAME: z.string().default("main"),
+
+ // Miscellaneous
+ IDMUX_URL: z.string().optional(),
+ GITHUB_RUN_NUMBER: z.string().optional(),
+ GITHUB_REF_NAME: z.string().optional(),
+ RESTRICTED_COUNTRIES: delimitedList(",").optional(),
+});
+
+export const config = configSchema.parse(process.env);
diff --git a/apps/api/src/controllers/__tests__/crawl.test.ts b/apps/api/src/controllers/__tests__/crawl.test.ts
index 8d6b539d8..f2b0966ad 100644
--- a/apps/api/src/controllers/__tests__/crawl.test.ts
+++ b/apps/api/src/controllers/__tests__/crawl.test.ts
@@ -1,4 +1,5 @@
import { crawlController } from "../v0/crawl";
+import { config } from "../../config";
import { Request, Response } from "express";
import { authenticateUser } from "../auth"; // Ensure this import is correct
import { createIdempotencyKey } from "../../services/idempotency/create";
@@ -21,7 +22,7 @@ describe("crawlController", () => {
const req = {
headers: {
"x-idempotency-key": await uuidv7(),
- Authorization: `Bearer ${process.env.TEST_API_KEY}`,
+ Authorization: `Bearer ${config.TEST_API_KEY}`,
},
body: {
url: "https://mendable.ai",
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 3fd5379db..b67125418 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -1,4 +1,5 @@
import { parseApi } from "../lib/parseApi";
+import { config } from "../config";
import { getRateLimiter } from "../services/rate-limiter";
import { AuthResponse, NotificationType, RateLimiterMode } from "../types";
import {
@@ -151,16 +152,13 @@ export async function getACUC(
mode === RateLimiterMode.ExtractStatus ||
mode === RateLimiterMode.ExtractAgentPreview;
- if (api_key === process.env.PREVIEW_TOKEN) {
+ if (api_key === config.PREVIEW_TOKEN) {
const acuc = mockPreviewACUC(api_key, isExtract);
acuc.is_extract = isExtract;
return acuc;
}
- if (
- process.env.USE_DB_AUTHENTICATION !== "true" &&
- !process.env.SUPABASE_ACUC_URL
- ) {
+ if (config.USE_DB_AUTHENTICATION !== true && !config.SUPABASE_ACUC_URL) {
const acuc = mockACUC();
acuc.is_extract = isExtract;
return acuc;
@@ -181,7 +179,7 @@ export async function getACUC(
let retries = 0;
const maxRetries = 5;
while (retries < maxRetries) {
- const client = !!process.env.SUPABASE_ACUC_URL
+ const client = !!config.SUPABASE_ACUC_URL
? supabase_acuc_only_service
: Math.random() > 2 / 3
? supabase_rr_service
@@ -289,10 +287,7 @@ export async function getACUCTeam(
return acuc;
}
- if (
- process.env.USE_DB_AUTHENTICATION !== "true" &&
- !process.env.SUPABASE_ACUC_URL
- ) {
+ if (config.USE_DB_AUTHENTICATION !== true && !config.SUPABASE_ACUC_URL) {
const acuc = mockACUC();
acuc.is_extract = isExtract;
return acuc;
@@ -314,7 +309,7 @@ export async function getACUCTeam(
const maxRetries = 5;
while (retries < maxRetries) {
- const client = !!process.env.SUPABASE_ACUC_URL
+ const client = !!config.SUPABASE_ACUC_URL
? supabase_acuc_only_service
: Math.random() > 2 / 3
? supabase_rr_service
@@ -396,7 +391,7 @@ export async function authenticateUser(
res,
mode?: RateLimiterMode,
): Promise<AuthResponse> {
- if (!!process.env.SUPABASE_ACUC_URL) {
+ if (!!config.SUPABASE_ACUC_URL) {
return supaAuthenticateUser(req, res, mode);
}
@@ -446,7 +441,7 @@ async function supaAuthenticateUser(
"Unauthenticated Playground calls are temporarily disabled due to abuse. Please sign up.",
);
}
- if (token == process.env.PREVIEW_TOKEN) {
+ if (token == config.PREVIEW_TOKEN) {
if (mode == RateLimiterMode.CrawlStatus) {
rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token);
} else if (mode == RateLimiterMode.ExtractStatus) {
@@ -487,8 +482,7 @@ async function supaAuthenticateUser(
);
}
- const team_endpoint_token =
- token === process.env.PREVIEW_TOKEN ? iptoken : teamId;
+ const team_endpoint_token = token === config.PREVIEW_TOKEN ? iptoken : teamId;
try {
await rateLimiter.consume(team_endpoint_token);
@@ -518,7 +512,7 @@ async function supaAuthenticateUser(
}
if (
- token === process.env.PREVIEW_TOKEN &&
+ token === config.PREVIEW_TOKEN &&
(mode === RateLimiterMode.Scrape ||
mode === RateLimiterMode.Preview ||
mode === RateLimiterMode.Map ||
@@ -537,7 +531,7 @@ async function supaAuthenticateUser(
// if (origin && origin.includes("firecrawl.dev")){
// return { success: true, team_id: "preview" };
// }
- // if(process.env.ENV !== "production") {
+ // if(config.ENV !== "production") {
// return { success: true, team_id: "preview" };
// }
diff --git a/apps/api/src/controllers/v0/admin/check-fire-engine.ts b/apps/api/src/controllers/v0/admin/check-fire-engine.ts
index f8bf8beb8..6f90e8850 100644
--- a/apps/api/src/controllers/v0/admin/check-fire-engine.ts
+++ b/apps/api/src/controllers/v0/admin/check-fire-engine.ts
@@ -1,10 +1,11 @@
import { logger } from "../../../lib/logger";
+import { config } from "../../../config";
import * as Sentry from "@sentry/node";
import { Request, Response } from "express";
export async function checkFireEngine(req: Request, res: Response) {
try {
- if (!process.env.FIRE_ENGINE_BETA_URL) {
+ if (!config.FIRE_ENGINE_BETA_URL) {
logger.warn("Fire engine beta URL not configured");
return res.status(500).json({
success: false,
@@ -20,21 +21,18 @@ export async function checkFireEngine(req: Request, res: Response) {
for (const url of urls) {
try {
- const response = await fetch(
- `${process.env.FIRE_ENGINE_BETA_URL}/scrape`,
- {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- "X-Disable-Cache": "true",
- },
- body: JSON.stringify({
- url,
- engine: "chrome-cdp",
- }),
- signal: controller.signal,
+ const response = await fetch(`${config.FIRE_ENGINE_BETA_URL}/scrape`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ "X-Disable-Cache": "true",
},
- );
+ body: JSON.stringify({
+ url,
+ engine: "chrome-cdp",
+ }),
+ signal: controller.signal,
+ });
clearTimeout(timeout);
diff --git a/apps/api/src/controllers/v0/admin/redis-health.ts b/apps/api/src/controllers/v0/admin/redis-health.ts
index c6f0419d2..53a281b82 100644
--- a/apps/api/src/controllers/v0/admin/redis-health.ts
+++ b/apps/api/src/controllers/v0/admin/redis-health.ts
@@ -1,4 +1,5 @@
import { Request, Response } from "express";
+import { config } from "../../../config";
import Redis from "ioredis";
import { logger } from "../../../lib/logger";
import { redisRateLimitClient } from "../../../services/rate-limiter";
@@ -17,7 +18,7 @@ export async function redisHealthController(req: Request, res: Response) {
};
try {
- const queueRedis = new Redis(process.env.REDIS_URL!);
+ const queueRedis = new Redis(config.REDIS_URL!);
const testKey = "test";
const testValue = "test";
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index aea5775d3..7262c4d58 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -1,4 +1,5 @@
import { Request, Response } from "express";
+import { config } from "../../config";
import { authenticateUser } from "../auth";
import { RateLimiterMode } from "../../../src/types";
import { redisEvictConnection } from "../../../src/services/redis";
@@ -20,10 +21,10 @@ async function getJobs(
): Promise<PseudoJob<any>[]> {
const [nuqJobs, dbScrapes, gcsJobs] = await Promise.all([
scrapeQueue.getJobs(ids),
- process.env.USE_DB_AUTHENTICATION === "true"
+ config.USE_DB_AUTHENTICATION
? await supabaseGetScrapesByRequestId(crawlId)
: [],
- process.env.GCS_BUCKET_NAME
+ config.GCS_BUCKET_NAME
? (Promise.all(
ids.map(async x => ({ id: x, job: await getJobFromGCS(x) })),
).then(x => x.filter(x => x.job)) as Promise<
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index c6936e068..435250be4 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import { v7 as uuidv7 } from "uuid";
import {
BatchScrapeRequest,
@@ -139,7 +140,7 @@ export async function batchScrapeController(
...internalOptions,
disableSmartWaitCache: true,
teamId: req.auth.team_id,
- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME
+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME
? true
: false,
zeroDataRetention,
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index 55e1ad837..ffe321330 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import {
CrawlErrorsResponse,
CrawlStatusParams,
@@ -72,7 +73,7 @@ export async function crawlErrorsController(
"crawl:" + req.params.jobId + ":robots_blocked",
),
});
- } else if (process.env.USE_DB_AUTHENTICATION === "true") {
+ } else if (config.USE_DB_AUTHENTICATION) {
// Check the requests table for the crawl/batch scrape request
const { data: request, error: requestError } = await supabase_rr_service
.from("requests")
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 7bb1c7b46..f39d4103a 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import {
CrawlStatusParams,
CrawlStatusResponse,
@@ -54,12 +55,10 @@ export type DBScrape = {
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const [nuqJob, dbScrape, gcsJob] = await Promise.all([
scrapeQueue.getJob(id) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,
- (process.env.USE_DB_AUTHENTICATION === "true"
+ (config.USE_DB_AUTHENTICATION
? supabaseGetScrapeById(id)
: null) as Promise<DBScrape | null>,
- (process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<
- any | null
- >,
+ (config.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
]);
if (!nuqJob && !dbScrape) return null;
@@ -98,10 +97,8 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const [nuqJobs, dbScrapes, gcsJobs] = await Promise.all([
scrapeQueue.getJobs(ids) as Promise<NuQJob<ScrapeJobSingleUrls>[]>,
- process.env.USE_DB_AUTHENTICATION === "true"
- ? supabaseGetScrapesById(ids)
- : [],
- process.env.GCS_BUCKET_NAME
+ config.USE_DB_AUTHENTICATION ? supabaseGetScrapesById(ids) : [],
+ config.GCS_BUCKET_NAME
? (Promise.all(
ids.map(async x => ({ id: x, job: await getJobFromGCS(x) })),
).then(x => x.filter(x => x.job)) as Promise<
@@ -198,16 +195,15 @@ export async function crawlStatusController(
logger.child({ zeroDataRetention }),
);
- const creditsRpc =
- process.env.USE_DB_AUTHENTICATION === "true"
- ? await supabase_service.rpc(
- "credits_billed_by_crawl_id_2",
- {
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- )
- : null;
+ const creditsRpc = config.USE_DB_AUTHENTICATION
+ ? await supabase_service.rpc(
+ "credits_billed_by_crawl_id_2",
+ {
+ i_crawl_id: req.params.jobId,
+ },
+ { get: true },
+ )
+ : null;
let outputBulkA: {
status?: "completed" | "scraping" | "cancelled";
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index b67415cd3..bd57db1bd 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import { v7 as uuidv7 } from "uuid";
import {
CrawlRequest,
@@ -65,7 +66,7 @@ export async function crawlController(
});
let { remainingCredits } = req.account!;
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;
if (!useDbAuthentication) {
remainingCredits = Infinity;
}
@@ -121,9 +122,7 @@ export async function crawlController(
...internalOptions,
disableSmartWaitCache: true,
teamId: req.auth.team_id,
- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME
- ? true
- : false,
+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
zeroDataRetention,
}, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter
team_id: req.auth.team_id,
diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.ts
index abfa06775..907819331 100644
--- a/apps/api/src/controllers/v1/extract-status.ts
+++ b/apps/api/src/controllers/v1/extract-status.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import { RequestWithAuth } from "./types";
import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";
import { getExtractQueue } from "../../services/queue-service";
@@ -33,7 +34,7 @@ async function getExtractJob(
): Promise<ExtractPseudoJob<ExtractResult> | null> {
const [bullJob, dbExtract] = await Promise.all([
getExtractQueue().getJob(id),
- (process.env.USE_DB_AUTHENTICATION === "true"
+ (config.USE_DB_AUTHENTICATION
? supabaseGetExtractByIdDirect(id)
: null) as Promise<DBExtract | null>,
]);
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index cd3eb1218..eb2d08e29 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -24,6 +24,7 @@ import {
import { createWebhookSender, WebhookEvent } from "../../services/webhook";
import { logRequest } from "../../services/logging/log_job";
+import { config } from "../../config";
async function oldExtract(
req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
res: Response<ExtractResponse>,
@@ -113,7 +114,7 @@ export async function extractController(
req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
res: Response<ExtractResponse>,
) {
- const selfHosted = process.env.USE_DB_AUTHENTICATION !== "true";
+ const selfHosted = config.USE_DB_AUTHENTICATION !== true;
const originalRequest = { ...req.body };
req.body = extractRequestSchema.parse(req.body);
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index ca514e954..772ae393d 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import { v7 as uuidv7 } from "uuid";
import {
MapDocument,
@@ -31,7 +32,7 @@ import { MapTimeoutError } from "../../lib/error";
import { checkPermissions } from "../../lib/permissions";
configDotenv();
-const redis = new Redis(process.env.REDIS_URL!);
+const redis = new Redis(config.REDIS_URL!);
// Max Links that "Smart /map" can return
const MAX_FIRE_ENGINE_RESULTS = 500;
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 47e850017..d7af937f6 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import { logger as _logger } from "../../lib/logger";
import {
Document,
@@ -81,8 +82,8 @@ export async function scrapeController(
// const startTime = new Date().getTime();
const isDirectToBullMQ =
- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+ config.SEARCH_PREVIEW_TOKEN !== undefined &&
+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(
req.body,
@@ -147,7 +148,7 @@ export async function scrapeController(
internalOptions: {
...internalOptions,
teamId: req.auth.team_id,
- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME
+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME
? true
: false,
unnormalizedSourceURL: preNormalizedBody.url,
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index f901a13fa..0fcacdc10 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import {
Document,
RequestWithAuth,
@@ -170,7 +171,7 @@ async function scrapeSearchResult(
};
let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;
- if (process.env.USE_DB_AUTHENTICATION === "true") {
+ if (config.USE_DB_AUTHENTICATION) {
const { data: costTrackingResponse, error: costTrackingError } =
await supabase_service
.from("scrapes")
@@ -256,8 +257,8 @@ export async function searchController(
};
const middlewareTime = controllerStartTime - middlewareStartTime;
const isSearchPreview =
- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+ config.SEARCH_PREVIEW_TOKEN !== undefined &&
+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
let credits_billed = 0;
let allDocsWithCostTracking: DocumentWithCostTracking[] = [];
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 1768632d3..ad360009f 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1,4 +1,5 @@
import { Request, Response } from "express";
+import { config } from "../../config";
import { z } from "zod";
import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
import { countries } from "../../lib/validate-country";
@@ -52,10 +53,7 @@ export const url = z.preprocess(
.url()
.regex(/^https?:\/\//i, "URL uses unsupported protocol")
.refine(x => {
- if (
- process.env.TEST_SUITE_SELF_HOSTED === "true" &&
- process.env.ALLOW_LOCAL_WEBHOOKS === "true"
- ) {
+ if (config.TEST_SUITE_SELF_HOSTED && config.ALLOW_LOCAL_WEBHOOKS) {
if (
/^https?:\/\/(localhost|127\.0\.0\.1|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})(:\d+)?([\/?#]|$)/i.test(
x as string,
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index 7cfb39530..ff5f9b0e2 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import {
Document,
RequestWithAuth,
@@ -120,7 +121,7 @@ async function scrapeX402SearchResult(
};
let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;
- if (process.env.USE_DB_AUTHENTICATION === "true") {
+ if (config.USE_DB_AUTHENTICATION) {
const { data: costTrackingResponse, error: costTrackingError } =
await supabase_service
.from("scrapes")
@@ -197,8 +198,8 @@ export async function x402SearchController(
};
const startTime = new Date().getTime();
const isSearchPreview =
- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+ config.SEARCH_PREVIEW_TOKEN !== undefined &&
+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
try {
req.body = searchRequestSchema.parse(req.body);
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index e530df0c3..30c503959 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import { v7 as uuidv7 } from "uuid";
import {
BatchScrapeRequest,
@@ -132,7 +133,7 @@ export async function batchScrapeController(
internalOptions: {
disableSmartWaitCache: true,
teamId: req.auth.team_id,
- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME
+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME
? true
: false,
zeroDataRetention,
diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts
index 55e1ad837..ffe321330 100644
--- a/apps/api/src/controllers/v2/crawl-errors.ts
+++ b/apps/api/src/controllers/v2/crawl-errors.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import {
CrawlErrorsResponse,
CrawlStatusParams,
@@ -72,7 +73,7 @@ export async function crawlErrorsController(
"crawl:" + req.params.jobId + ":robots_blocked",
),
});
- } else if (process.env.USE_DB_AUTHENTICATION === "true") {
+ } else if (config.USE_DB_AUTHENTICATION) {
// Check the requests table for the crawl/batch scrape request
const { data: request, error: requestError } = await supabase_rr_service
.from("requests")
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 077fb565f..9d29491e0 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import {
CrawlStatusParams,
CrawlStatusResponse,
@@ -62,12 +63,10 @@ export async function getJob(
id,
_logger,
) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,
- (process.env.USE_DB_AUTHENTICATION === "true"
+ (config.USE_DB_AUTHENTICATION
? supabaseGetScrapeById(id)
: null) as Promise<DBScrape | null>,
- (process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<
- any | null
- >,
+ (config.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
]);
if (!nuqJob && !dbScrape) return null;
@@ -109,10 +108,8 @@ export async function getJobs(
): Promise<PseudoJob<any>[]> {
const [nuqJobs, dbScrapes, gcsJobs] = await Promise.all([
scrapeQueue.getJobs(ids, _logger) as Promise<NuQJob<ScrapeJobSingleUrls>[]>,
- process.env.USE_DB_AUTHENTICATION === "true"
- ? supabaseGetScrapesById(ids)
- : [],
- process.env.GCS_BUCKET_NAME
+ config.USE_DB_AUTHENTICATION ? supabaseGetScrapesById(ids) : [],
+ config.GCS_BUCKET_NAME
? (Promise.all(
ids.map(async x => ({ id: x, job: await getJobFromGCS(x) })),
).then(x => x.filter(x => x.job)) as Promise<
@@ -218,16 +215,15 @@ export async function crawlStatusController(
logger.child({ zeroDataRetention }),
);
- const creditsRpc =
- process.env.USE_DB_AUTHENTICATION === "true"
- ? await supabase_service.rpc(
- "credits_billed_by_crawl_id_2",
- {
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- )
- : null;
+ const creditsRpc = config.USE_DB_AUTHENTICATION
+ ? await supabase_service.rpc(
+ "credits_billed_by_crawl_id_2",
+ {
+ i_crawl_id: req.params.jobId,
+ },
+ { get: true },
+ )
+ : null;
let outputBulkA: {
status?: "completed" | "scraping" | "cancelled";
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index babd3a0dd..4534e01fa 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import { v7 as uuidv7 } from "uuid";
import {
CrawlRequest,
@@ -72,7 +73,7 @@ export async function crawlController(
});
let { remainingCredits } = req.account!;
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;
if (!useDbAuthentication) {
remainingCredits = Infinity;
}
@@ -185,9 +186,7 @@ export async function crawlController(
internalOptions: {
disableSmartWaitCache: true,
teamId: req.auth.team_id,
- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME
- ? true
- : false,
+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
zeroDataRetention,
},
team_id: req.auth.team_id,
diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.ts
index 3c55c14e6..378554c0a 100644
--- a/apps/api/src/controllers/v2/extract-status.ts
+++ b/apps/api/src/controllers/v2/extract-status.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import { RequestWithAuth } from "./types";
import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";
import { getExtractQueue } from "../../services/queue-service";
@@ -33,7 +34,7 @@ async function getExtractJob(
): Promise<ExtractPseudoJob<ExtractResult> | null> {
const [bullJob, dbExtract] = await Promise.all([
getExtractQueue().getJob(id),
- (process.env.USE_DB_AUTHENTICATION === "true"
+ (config.USE_DB_AUTHENTICATION
? supabaseGetExtractByIdDirect(id)
: null) as Promise<DBExtract | null>,
]);
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 78c005902..6fd95c75e 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import { logger as _logger } from "../../lib/logger";
import {
Document,
@@ -120,8 +121,8 @@ export async function scrapeController(
const timeout = req.body.timeout;
const isDirectToBullMQ =
- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+ config.SEARCH_PREVIEW_TOKEN !== undefined &&
+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
const totalWait =
(req.body.waitFor ?? 0) +
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 98f171084..a85924c51 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import {
Document,
RequestWithAuth,
@@ -161,7 +162,7 @@ async function scrapeSearchResult(
};
let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;
- if (process.env.USE_DB_AUTHENTICATION === "true") {
+ if (config.USE_DB_AUTHENTICATION) {
const { data: costTrackingResponse, error: costTrackingError } =
await supabase_service
.from("scrapes")
@@ -236,8 +237,8 @@ export async function searchController(
const middlewareTime = controllerStartTime - middlewareStartTime;
const isSearchPreview =
- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+ config.SEARCH_PREVIEW_TOKEN !== undefined &&
+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
let credits_billed = 0;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index c3ff9869c..c5de2d7c5 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1,4 +1,5 @@
import { Request, Response } from "express";
+import { config } from "../../config";
import { z } from "zod";
import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
import { countries } from "../../lib/validate-country";
@@ -46,10 +47,7 @@ const BASE_URL_SCHEMA = z.preprocess(
.url()
.regex(/^https?:\/\//i, "URL uses unsupported protocol")
.refine(x => {
- if (
- process.env.TEST_SUITE_SELF_HOSTED === "true" &&
- process.env.ALLOW_LOCAL_WEBHOOKS === "true"
- ) {
+ if (config.TEST_SUITE_SELF_HOSTED && config.ALLOW_LOCAL_WEBHOOKS) {
if (
/^https?:\/\/(localhost|127\.0\.0\.1|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})(:\d+)?([\/?#]|$)/i.test(
x as string,
diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.ts
index 81e6fdfd0..bc09fb7cb 100644
--- a/apps/api/src/controllers/v2/x402-search.ts
+++ b/apps/api/src/controllers/v2/x402-search.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { config } from "../../config";
import {
Document,
RequestWithAuth,
@@ -148,7 +149,7 @@ async function scrapeX420SearchResult(
};
let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;
- if (process.env.USE_DB_AUTHENTICATION === "true") {
+ if (config.USE_DB_AUTHENTICATION) {
const { data: costTrackingResponse, error: costTrackingError } =
await supabase_service
.from("scrapes")
@@ -218,8 +219,8 @@ export async function x402SearchController(
const startTime = new Date().getTime();
const isSearchPreview =
- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+ config.SEARCH_PREVIEW_TOKEN !== undefined &&
+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
let credits_billed = 0;
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index 56a8b4c9a..a3b43697b 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -1,4 +1,4 @@
-import "dotenv/config";
+import { config } from "./config";
import { type ChildProcess, spawn } from "child_process";
import * as net from "net";
import { basename, join } from "path";
@@ -87,21 +87,19 @@ function formatDuration(nanoseconds: bigint): string {
return `${minutes}m ${remainingSeconds.toFixed(0)}s`;
}
-const PORT = process.env.PORT ?? "3002";
-const WORKER_PORT = process.env.WORKER_PORT ?? "3005";
-const EXTRACT_WORKER_PORT = process.env.EXTRACT_WORKER_PORT ?? "3004";
-const NUQ_WORKER_START_PORT = Number(
- process.env.NUQ_WORKER_START_PORT ?? "3006",
-);
-const NUQ_WORKER_COUNT = Number(process.env.NUQ_WORKER_COUNT ?? "5");
+const PORT = config.PORT;
+const WORKER_PORT = config.WORKER_PORT;
+const EXTRACT_WORKER_PORT = config.EXTRACT_WORKER_PORT;
+const NUQ_WORKER_START_PORT = config.NUQ_WORKER_START_PORT;
+const NUQ_WORKER_COUNT = config.NUQ_WORKER_COUNT;
const NUQ_PREFETCH_WORKER_PORT = NUQ_WORKER_START_PORT + NUQ_WORKER_COUNT;
// PostgreSQL credentials (with defaults for backward compatibility)
-const POSTGRES_USER = process.env.POSTGRES_USER ?? "postgres";
-const POSTGRES_PASSWORD = process.env.POSTGRES_PASSWORD ?? "postgres";
-const POSTGRES_DB = process.env.POSTGRES_DB ?? "postgres";
-const POSTGRES_HOST = process.env.POSTGRES_HOST ?? "localhost";
-const POSTGRES_PORT = process.env.POSTGRES_PORT ?? "5432";
+const POSTGRES_USER = config.POSTGRES_USER;
+const POSTGRES_PASSWORD = config.POSTGRES_PASSWORD;
+const POSTGRES_DB = config.POSTGRES_DB;
+const POSTGRES_HOST = config.POSTGRES_HOST;
+const POSTGRES_PORT = config.POSTGRES_PORT;
// Shell escape helper to prevent command injection
function shellEscape(arg: string): string {
@@ -492,7 +490,7 @@ async function waitForPostgres(
async function setupNuqPostgres(): Promise<Services["nuqPostgres"]> {
// If NUQ_DATABASE_URL is already set, respect it (user's explicit choice)
- if (process.env.NUQ_DATABASE_URL) {
+ if (config.NUQ_DATABASE_URL) {
logger.info("NUQ_DATABASE_URL is set, skipping container management");
return undefined;
}
@@ -504,6 +502,8 @@ async function setupNuqPostgres(): Promise<Services["nuqPostgres"]> {
// Running in docker-compose: construct URL with proper encoding
logger.section("Setting up NUQ PostgreSQL connection for docker-compose");
const dbUrl = `postgresql://${encodeURIComponent(POSTGRES_USER)}:${encodeURIComponent(POSTGRES_PASSWORD)}@${POSTGRES_HOST}:${POSTGRES_PORT}/${encodeURIComponent(POSTGRES_DB)}`;
+ config.NUQ_DATABASE_URL = dbUrl;
+ config.NUQ_DATABASE_URL_LISTEN = dbUrl;
process.env.NUQ_DATABASE_URL = dbUrl;
process.env.NUQ_DATABASE_URL_LISTEN = dbUrl;
logger.success(
@@ -540,6 +540,8 @@ async function setupNuqPostgres(): Promise<Services["nuqPostgres"]> {
// Set environment variables for the services with proper encoding
const dbUrl = `postgresql://${encodeURIComponent(POSTGRES_USER)}:${encodeURIComponent(POSTGRES_PASSWORD)}@localhost:5432/${encodeURIComponent(POSTGRES_DB)}`;
+ config.NUQ_DATABASE_URL = dbUrl;
+ config.NUQ_DATABASE_URL_LISTEN = dbUrl;
process.env.NUQ_DATABASE_URL = dbUrl;
process.env.NUQ_DATABASE_URL_LISTEN = dbUrl;
@@ -623,7 +625,7 @@ async function startServices(command?: string[]): Promise<Services> {
{
NUQ_REDUCE_NOISE: "true",
NUQ_POD_NAME: "worker",
- WORKER_PORT: WORKER_PORT,
+ WORKER_PORT: String(WORKER_PORT),
},
);
@@ -635,7 +637,7 @@ async function startServices(command?: string[]): Promise<Services> {
{
NUQ_REDUCE_NOISE: "true",
NUQ_POD_NAME: "extract-worker",
- EXTRACT_WORKER_PORT: EXTRACT_WORKER_PORT,
+ EXTRACT_WORKER_PORT: String(EXTRACT_WORKER_PORT),
},
);
@@ -653,7 +655,7 @@ async function startServices(command?: string[]): Promise<Services> {
),
);
- const nuqPrefetchWorker = process.env.NUQ_RABBITMQ_URL
+ const nuqPrefetchWorker = config.NUQ_RABBITMQ_URL
? execForward(
"nuq-prefetch-worker",
process.argv[2] === "--start-docker"
@@ -668,19 +670,18 @@ async function startServices(command?: string[]): Promise<Services> {
)
: undefined;
- const indexWorker =
- process.env.USE_DB_AUTHENTICATION === "true"
- ? execForward(
- "index-worker",
- process.argv[2] === "--start-docker"
- ? "node dist/src/services/indexing/index-worker.js"
- : "pnpm index-worker:production",
- {
- NUQ_REDUCE_NOISE: "true",
- NUQ_POD_NAME: "index-worker",
- },
- )
- : undefined;
+ const indexWorker = config.USE_DB_AUTHENTICATION
+ ? execForward(
+ "index-worker",
+ process.argv[2] === "--start-docker"
+ ? "node dist/src/services/indexing/index-worker.js"
+ : "pnpm index-worker:production",
+ {
+ NUQ_REDUCE_NOISE: "true",
+ NUQ_POD_NAME: "index-worker",
+ },
+ )
+ : undefined;
// tests hammer the API instantly, so we need to ensure it's running before launching tests
if (
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 1b227273c..4e9a6b5b2 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -1,4 +1,5 @@
import "dotenv/config";
+import { config } from "./config";
import "./services/sentry";
import { setSentryServiceTag } from "./services/sentry";
import * as Sentry from "@sentry/node";
@@ -41,7 +42,7 @@ const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");
const { ExpressAdapter } = require("@bull-board/express");
-const numCPUs = process.env.ENV === "local" ? 2 : os.cpus().length;
+const numCPUs = config.ENV === "local" ? 2 : os.cpus().length;
logger.info(`Number of CPUs: ${numCPUs} available`);
logger.info("Network info dump", {
@@ -57,7 +58,7 @@ const expressApp = express();
const ws = expressWs(expressApp);
const app = ws.app;
-global.isProduction = process.env.IS_PRODUCTION === "true";
+global.isProduction = config.IS_PRODUCTION;
setSentryServiceTag("api");
@@ -70,12 +71,12 @@ app.use(responseTime());
app.disable("x-powered-by");
-if (process.env.EXPRESS_TRUST_PROXY) {
- app.set("trust proxy", parseInt(process.env.EXPRESS_TRUST_PROXY, 10));
+if (config.EXPRESS_TRUST_PROXY) {
+ app.set("trust proxy", config.EXPRESS_TRUST_PROXY);
}
const serverAdapter = new ExpressAdapter();
-serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);
+serverAdapter.setBasePath(`/admin/${config.BULL_AUTH_KEY}/queues`);
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: [
@@ -88,10 +89,7 @@ const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
serverAdapter: serverAdapter,
});
-app.use(
- `/admin/${process.env.BULL_AUTH_KEY}/queues`,
- serverAdapter.getRouter(),
-);
+app.use(`/admin/${config.BULL_AUTH_KEY}/queues`, serverAdapter.getRouter());
app.get("/", (_, res) => {
res.redirect("https://docs.firecrawl.dev/api-reference/v2-introduction");
@@ -108,8 +106,8 @@ app.use("/v2", v2Router);
app.use(adminRouter);
app.use(domainFrequencyRouter);
-const DEFAULT_PORT = process.env.PORT ?? 3002;
-const HOST = process.env.HOST ?? "localhost";
+const DEFAULT_PORT = config.PORT;
+const HOST = config.HOST;
async function startServer(port = DEFAULT_PORT) {
try {
@@ -131,7 +129,7 @@ async function startServer(port = DEFAULT_PORT) {
const exitHandler = async () => {
logger.info("SIGTERM signal received: closing HTTP server");
- if (process.env.IS_KUBERNETES === "true") {
+ if (config.IS_KUBERNETES) {
// Account for GCE load balancer drain timeout
logger.info("Waiting 60s for GCE load balancer drain timeout");
await new Promise(resolve => setTimeout(resolve, 60000));
diff --git a/apps/api/src/lib/branding/transformer.ts b/apps/api/src/lib/branding/transformer.ts
index 5e64dcf97..8a6471448 100644
--- a/apps/api/src/lib/branding/transformer.ts
+++ b/apps/api/src/lib/branding/transformer.ts
@@ -1,4 +1,5 @@
import { processRawBranding } from "./processor";
+import { config } from "../../config";
import { BrandingProfile } from "../../types/branding";
import { enhanceBrandingWithLLM } from "./llm";
import { Meta } from "../../scraper/scrapeURL";
@@ -94,7 +95,7 @@ export async function brandingTransformer(
brandingProfile = jsBranding;
}
- if (process.env.DEBUG_BRANDING !== "true") {
+ if (config.DEBUG_BRANDING !== true) {
delete (brandingProfile as any).__button_snapshots;
}
diff --git a/apps/api/src/lib/deployment.ts b/apps/api/src/lib/deployment.ts
index e83e4063e..0e3ccc539 100644
--- a/apps/api/src/lib/deployment.ts
+++ b/apps/api/src/lib/deployment.ts
@@ -1,5 +1,6 @@
+import { config } from "../config";
export function isSelfHosted(): boolean {
- return process.env.USE_DB_AUTHENTICATION !== "true";
+ return config.USE_DB_AUTHENTICATION !== true;
}
export function getErrorContactMessage(errorId?: string): string {
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index d5d702143..c91765748 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -1,4 +1,5 @@
import { v7 as uuidv7 } from "uuid";
+import { config } from "../../config";
import {
Document,
ScrapeOptions,
@@ -59,7 +60,7 @@ export async function scrapeDocument(
}),
internalOptions: {
teamId: options.teamId,
- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME
+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME
? true
: false,
bypassBilling: true,
diff --git a/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts b/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts
index ab6db63f6..04d9e5c09 100644
--- a/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/usage/llm-cost-f0.ts
@@ -1,4 +1,5 @@
import { TokenUsage } from "../../../../controllers/v1/types";
+import { config } from "../../../../config";
import { logger } from "../../../../lib/logger";
import { modelPrices } from "../../usage/model-prices";
@@ -26,7 +27,7 @@ export function estimateTotalCost_F0(tokenUsage: TokenUsage[]): number {
export function estimateCost_F0(tokenUsage: TokenUsage): number {
let totalCost = 0;
try {
- let model = tokenUsage.model ?? (process.env.MODEL_NAME || "gpt-4o-mini");
+ let model = tokenUsage.model ?? config.MODEL_NAME;
const pricing = modelPrices[model] as ModelPricing;
if (!pricing) {
diff --git a/apps/api/src/lib/extract/usage/llm-cost.ts b/apps/api/src/lib/extract/usage/llm-cost.ts
index 4a9fd3a95..a24b4ff4c 100644
--- a/apps/api/src/lib/extract/usage/llm-cost.ts
+++ b/apps/api/src/lib/extract/usage/llm-cost.ts
@@ -1,4 +1,5 @@
import { TokenUsage } from "../../../controllers/v1/types";
+import { config } from "../../../config";
import { logger } from "../../../lib/logger";
import { CostTracking } from "../../cost-tracking";
import { modelPrices } from "./model-prices";
@@ -31,7 +32,7 @@ export function estimateTotalCost(tokenUsage: TokenUsage[]): number {
function estimateCost(tokenUsage: TokenUsage): number {
let totalCost = 0;
try {
- let model = tokenUsage.model ?? (process.env.MODEL_NAME || "gpt-4o-mini");
+ let model = tokenUsage.model ?? config.MODEL_NAME;
const pricing = modelPrices[model] as ModelPricing;
if (!pricing) {
diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.ts
index 62bdd0280..6896c5914 100644
--- a/apps/api/src/lib/gcs-jobs.ts
+++ b/apps/api/src/lib/gcs-jobs.ts
@@ -10,9 +10,10 @@ import type {
LoggedScrape,
LoggedSearch,
} from "../services/logging/log_job";
+import { config } from "../config";
-const credentials = process.env.GCS_CREDENTIALS
- ? JSON.parse(atob(process.env.GCS_CREDENTIALS))
+const credentials = config.GCS_CREDENTIALS
+ ? JSON.parse(atob(config.GCS_CREDENTIALS))
: undefined;
export const storage = new Storage({ credentials });
@@ -27,12 +28,12 @@ export async function saveScrapeToGCS(scrape: LoggedScrape): Promise<void> {
"job.num_docs": 1,
});
- if (!process.env.GCS_BUCKET_NAME) {
+ if (!config.GCS_BUCKET_NAME) {
setSpanAttributes(span, { "gcs.bucket_configured": false });
return;
}
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${scrape.id}.json`);
// Save job docs with retry
@@ -119,12 +120,12 @@ export async function saveSearchToGCS(search: LoggedSearch): Promise<void> {
request_id: search.request_id,
});
- if (!process.env.GCS_BUCKET_NAME) {
+ if (!config.GCS_BUCKET_NAME) {
setSpanAttributes(span, { "gcs.bucket_configured": false });
return;
}
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${search.id}.json`);
for (let i = 0; i < 3; i++) {
@@ -197,12 +198,12 @@ export async function saveExtractToGCS(extract: LoggedExtract): Promise<void> {
"extract.team_id": extract.team_id,
});
- if (!process.env.GCS_BUCKET_NAME) {
+ if (!config.GCS_BUCKET_NAME) {
setSpanAttributes(span, { "gcs.bucket_configured": false });
return;
}
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${extract.id}.json`);
for (let i = 0; i < 3; i++) {
@@ -271,12 +272,12 @@ export async function saveMapToGCS(map: LoggedMap): Promise<void> {
"map.team_id": map.team_id,
});
- if (!process.env.GCS_BUCKET_NAME) {
+ if (!config.GCS_BUCKET_NAME) {
setSpanAttributes(span, { "gcs.bucket_configured": false });
return;
}
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${map.id}.json`);
for (let i = 0; i < 3; i++) {
@@ -345,12 +346,12 @@ export async function saveDeepResearchToGCS(
"deep_research.team_id": deepResearch.team_id,
});
- if (!process.env.GCS_BUCKET_NAME) {
+ if (!config.GCS_BUCKET_NAME) {
setSpanAttributes(span, { "gcs.bucket_configured": false });
return;
}
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${deepResearch.id}.json`);
for (let i = 0; i < 3; i++) {
@@ -418,12 +419,12 @@ export async function saveLlmsTxtToGCS(llmsTxt: LoggedLlmsTxt): Promise<void> {
"llms_txt.team_id": llmsTxt.team_id,
});
- if (!process.env.GCS_BUCKET_NAME) {
+ if (!config.GCS_BUCKET_NAME) {
setSpanAttributes(span, { "gcs.bucket_configured": false });
return;
}
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${llmsTxt.id}.json`);
for (let i = 0; i < 3; i++) {
@@ -491,12 +492,12 @@ export async function getJobFromGCS(jobId: string): Promise<Document[] | null> {
"job.id": jobId,
});
- if (!process.env.GCS_BUCKET_NAME) {
+ if (!config.GCS_BUCKET_NAME) {
setSpanAttributes(span, { "gcs.bucket_configured": false });
return null;
}
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${jobId}.json`);
try {
@@ -531,12 +532,12 @@ export async function removeJobFromGCS(jobId: string): Promise<void> {
"job.id": jobId,
});
- if (!process.env.GCS_BUCKET_NAME) {
+ if (!config.GCS_BUCKET_NAME) {
setSpanAttributes(span, { "gcs.bucket_configured": false });
return;
}
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${jobId}.json`);
try {
@@ -570,11 +571,11 @@ export async function getDocFromGCS(url: string): Promise<any | null> {
// url,
// });
try {
- if (!process.env.GCS_FIRE_ENGINE_BUCKET_NAME) {
+ if (!config.GCS_FIRE_ENGINE_BUCKET_NAME) {
return null;
}
- const bucket = storage.bucket(process.env.GCS_FIRE_ENGINE_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_FIRE_ENGINE_BUCKET_NAME);
const blob = bucket.file(`${url}`);
const [exists] = await blob.exists();
if (!exists) {
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index 5f42a5734..0e7273559 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -1,9 +1,10 @@
import { logger } from "./logger";
+import { config } from "../config";
import crypto from "crypto";
import { storage } from "./gcs-jobs";
-const credentials = process.env.GCS_CREDENTIALS
- ? JSON.parse(atob(process.env.GCS_CREDENTIALS))
+const credentials = config.GCS_CREDENTIALS
+ ? JSON.parse(atob(config.GCS_CREDENTIALS))
: undefined;
const PDF_CACHE_PREFIX = "pdf-cache-v2/";
@@ -23,12 +24,12 @@ export async function savePdfResultToCache(
result: { markdown: string; html: string },
): Promise<string | null> {
try {
- if (!process.env.GCS_BUCKET_NAME) {
+ if (!config.GCS_BUCKET_NAME) {
return null;
}
const cacheKey = createPdfCacheKey(pdfContent);
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);
for (let i = 0; i < 3; i++) {
@@ -79,12 +80,12 @@ export async function getPdfResultFromCache(
pdfContent: string,
): Promise<{ markdown: string; html: string } | null> {
try {
- if (!process.env.GCS_BUCKET_NAME) {
+ if (!config.GCS_BUCKET_NAME) {
return null;
}
const cacheKey = createPdfCacheKey(pdfContent);
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);
const [exists] = await blob.exists();
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts
index eb37f6dce..a7372e3fe 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts
@@ -1,4 +1,5 @@
import { supabase_service } from "../../services/supabase";
+import { config } from "../../config";
import { logger } from "../logger";
import { normalizeUrl, normalizeUrlOnlyHostname } from "../canonical-url";
@@ -13,7 +14,7 @@ export async function getLlmsTextFromCache(
url: string,
maxUrls: number,
): Promise<LlmsTextCache | null> {
- if (process.env.USE_DB_AUTHENTICATION !== "true") {
+ if (config.USE_DB_AUTHENTICATION !== true) {
return null;
}
@@ -54,7 +55,7 @@ export async function saveLlmsTextToCache(
llmstxt_full: string,
maxUrls: number,
): Promise<void> {
- if (process.env.USE_DB_AUTHENTICATION !== "true") {
+ if (config.USE_DB_AUTHENTICATION !== true) {
return;
}
diff --git a/apps/api/src/lib/generic-ai.ts b/apps/api/src/lib/generic-ai.ts
index 85192990f..cc86e5e46 100644
--- a/apps/api/src/lib/generic-ai.ts
+++ b/apps/api/src/lib/generic-ai.ts
@@ -1,4 +1,5 @@
import { createOpenAI } from "@ai-sdk/openai";
+import { config } from "../config";
import { createOllama } from "ollama-ai-provider";
import { anthropic } from "@ai-sdk/anthropic";
import { groq } from "@ai-sdk/groq";
@@ -18,23 +19,21 @@ type Provider =
| "fireworks"
| "deepinfra"
| "vertex";
-const defaultProvider: Provider = process.env.OLLAMA_BASE_URL
- ? "ollama"
- : "openai";
+const defaultProvider: Provider = config.OLLAMA_BASE_URL ? "ollama" : "openai";
const providerList: Record<Provider, any> = {
openai: createOpenAI({
- apiKey: process.env.OPENAI_API_KEY,
- baseURL: process.env.OPENAI_BASE_URL,
+ apiKey: config.OPENAI_API_KEY,
+ baseURL: config.OPENAI_BASE_URL,
}), //OPENAI_API_KEY
ollama: createOllama({
- baseURL: process.env.OLLAMA_BASE_URL,
+ baseURL: config.OLLAMA_BASE_URL,
}),
anthropic, //ANTHROPIC_API_KEY
groq, //GROQ_API_KEY
google, //GOOGLE_GENERATIVE_AI_API_KEY
openrouter: createOpenRouter({
- apiKey: process.env.OPENROUTER_API_KEY,
+ apiKey: config.OPENROUTER_API_KEY,
}),
fireworks, //FIREWORKS_API_KEY
deepinfra, //DEEPINFRA_API_KEY
@@ -44,9 +43,9 @@ const providerList: Record<Provider, any> = {
baseURL:
"https://aiplatform.googleapis.com/v1/projects/firecrawl/locations/global/publishers/google",
location: "global",
- googleAuthOptions: process.env.VERTEX_CREDENTIALS
+ googleAuthOptions: config.VERTEX_CREDENTIALS
? {
- credentials: JSON.parse(atob(process.env.VERTEX_CREDENTIALS)),
+ credentials: JSON.parse(atob(config.VERTEX_CREDENTIALS)),
}
: {
keyFile: "./gke-key.json",
@@ -58,8 +57,8 @@ export function getModel(name: string, provider: Provider = defaultProvider) {
if (name === "gemini-2.5-pro") {
name = "gemini-2.5-pro";
}
- return process.env.MODEL_NAME
- ? providerList[provider](process.env.MODEL_NAME)
+ return config.MODEL_NAME
+ ? providerList[provider](config.MODEL_NAME)
: providerList[provider](name);
}
@@ -67,7 +66,7 @@ export function getEmbeddingModel(
name: string,
provider: Provider = defaultProvider,
) {
- return process.env.MODEL_EMBEDDING_NAME
- ? providerList[provider].embedding(process.env.MODEL_EMBEDDING_NAME)
+ return config.MODEL_EMBEDDING_NAME
+ ? providerList[provider].embedding(config.MODEL_EMBEDDING_NAME)
: providerList[provider].embedding(name);
}
diff --git a/apps/api/src/lib/html-to-markdown-client.ts b/apps/api/src/lib/html-to-markdown-client.ts
index 0541f4234..aca0a676b 100644
--- a/apps/api/src/lib/html-to-markdown-client.ts
+++ b/apps/api/src/lib/html-to-markdown-client.ts
@@ -6,6 +6,7 @@
*/
import axios, { AxiosInstance, AxiosError } from "axios";
+import { config } from "../config";
import { logger } from "./logger";
import * as Sentry from "@sentry/node";
@@ -38,10 +39,7 @@ export async function convertHTMLToMarkdownWithHttpService(
return "";
}
- const url =
- serviceUrl ||
- process.env.HTML_TO_MARKDOWN_SERVICE_URL ||
- "http://localhost:8080";
+ const url = serviceUrl || config.HTML_TO_MARKDOWN_SERVICE_URL;
const startTime = Date.now();
try {
diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts
index 04b162bda..b7d2f3848 100644
--- a/apps/api/src/lib/html-to-markdown.ts
+++ b/apps/api/src/lib/html-to-markdown.ts
@@ -1,14 +1,12 @@
import koffi from "koffi";
+import { config } from "../config";
import "../services/sentry";
import * as Sentry from "@sentry/node";
-
-import dotenv from "dotenv";
import { logger } from "./logger";
import { stat } from "fs/promises";
import { HTML_TO_MARKDOWN_PATH } from "../natives";
import { convertHTMLToMarkdownWithHttpService } from "./html-to-markdown-client";
import { postProcessMarkdown } from "@mendable/firecrawl-rs";
-dotenv.config();
// TODO: add a timeout to the Go parser
@@ -60,7 +58,7 @@ export async function parseMarkdown(
}
// Try HTTP service first if enabled
- if (process.env.HTML_TO_MARKDOWN_SERVICE_URL) {
+ if (config.HTML_TO_MARKDOWN_SERVICE_URL) {
try {
let markdownContent = await convertHTMLToMarkdownWithHttpService(html);
markdownContent = await postProcessMarkdown(markdownContent);
@@ -79,7 +77,7 @@ export async function parseMarkdown(
}
try {
- if (process.env.USE_GO_MARKDOWN_PARSER == "true") {
+ if (config.USE_GO_MARKDOWN_PARSER) {
const converter = await GoMarkdownConverter.getInstance();
let markdownContent = await converter.convertHTMLToMarkdown(html);
markdownContent = await postProcessMarkdown(markdownContent);
diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.ts
index 9c61cb9a2..ad6052c09 100644
--- a/apps/api/src/lib/logger.ts
+++ b/apps/api/src/lib/logger.ts
@@ -1,5 +1,6 @@
import * as winston from "winston";
+import { config } from "../config";
import { configDotenv } from "dotenv";
configDotenv();
@@ -36,7 +37,7 @@ const zeroDataRetentionFilter = winston.format(info => {
})();
export const logger = winston.createLogger({
- level: process.env.LOGGING_LEVEL?.toLowerCase() ?? "debug",
+ level: config.LOGGING_LEVEL?.toLowerCase() ?? "debug",
format: winston.format.json({
replacer(key, value) {
if (value instanceof Error) {
@@ -53,7 +54,7 @@ export const logger = winston.createLogger({
},
}),
transports: [
- ...(process.env.FIRECRAWL_LOG_TO_FILE
+ ...(config.FIRECRAWL_LOG_TO_FILE
? [
new winston.transports.File({
filename:
@@ -77,9 +78,9 @@ export const logger = winston.createLogger({
winston.format.metadata({
fillExcept: ["message", "level", "timestamp"],
}),
- ...((process.env.ENV === "production" &&
- process.env.SENTRY_ENVIRONMENT === "dev") ||
- process.env.ENV !== "production"
+ ...((config.ENV === "production" &&
+ config.SENTRY_ENVIRONMENT === "dev") ||
+ config.ENV !== "production"
? [winston.format.colorize(), logFormat]
: []),
),
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index 6a53018d3..ab2194f24 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -1,4 +1,5 @@
import robotsParser, { Robot } from "robots-parser";
+import { config } from "../config";
import { Logger } from "winston";
import { ScrapeOptions, scrapeOptions } from "../controllers/v2/types";
import { scrapeURL } from "../scraper/scrapeURL";
@@ -6,8 +7,8 @@ import { Engine } from "../scraper/scrapeURL/engines";
import { CostTracking } from "./cost-tracking";
const useFireEngine =
- process.env.FIRE_ENGINE_BETA_URL !== "" &&
- process.env.FIRE_ENGINE_BETA_URL !== undefined;
+ config.FIRE_ENGINE_BETA_URL !== "" &&
+ config.FIRE_ENGINE_BETA_URL !== undefined;
interface RobotsTxtChecker {
robotsTxtUrl: string;
diff --git a/apps/api/src/lib/search-index-client.ts b/apps/api/src/lib/search-index-client.ts
index dd0f1da67..6423accde 100644
--- a/apps/api/src/lib/search-index-client.ts
+++ b/apps/api/src/lib/search-index-client.ts
@@ -6,6 +6,7 @@
*/
import { logger as _logger } from "./logger";
+import { config } from "../config";
import type { Logger } from "winston";
interface SearchIndexClientConfig {
@@ -86,10 +87,7 @@ export class SearchIndexClient {
* Check if search index service is enabled
*/
static isEnabled(): boolean {
- return !!(
- process.env.SEARCH_SERVICE_URL &&
- process.env.ENABLE_SEARCH_INDEX === "true"
- );
+ return !!(config.SEARCH_SERVICE_URL && config.ENABLE_SEARCH_INDEX);
}
/**
@@ -321,8 +319,8 @@ export function getSearchIndexClient(): SearchIndexClient | null {
}
if (!searchIndexClient) {
- const baseUrl = process.env.SEARCH_SERVICE_URL!;
- const apiSecret = process.env.SEARCH_SERVICE_API_SECRET;
+ const baseUrl = config.SEARCH_SERVICE_URL!;
+ const apiSecret = config.SEARCH_SERVICE_API_SECRET;
searchIndexClient = new SearchIndexClient({
baseUrl,
diff --git a/apps/api/src/lib/withAuth.ts b/apps/api/src/lib/withAuth.ts
index bec3d4d1a..a9d602cbd 100644
--- a/apps/api/src/lib/withAuth.ts
+++ b/apps/api/src/lib/withAuth.ts
@@ -2,6 +2,7 @@ import { AuthResponse } from "../../src/types";
import { logger } from "./logger";
import * as Sentry from "@sentry/node";
import { configDotenv } from "dotenv";
+import { config } from "../config";
configDotenv();
let warningCount = 0;
@@ -11,7 +12,7 @@ export function withAuth<T, U extends any[]>(
mockSuccess: T,
) {
return async function (...args: U): Promise<T> {
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;
if (!useDbAuthentication) {
if (warningCount < 5) {
logger.warn("You're bypassing authentication");
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index 723d35ec3..cc6839595 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -1,4 +1,5 @@
import express from "express";
+import { config } from "../config";
import { redisHealthController } from "../controllers/v0/admin/redis-health";
import { wrap } from "./shared";
import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";
@@ -19,57 +20,54 @@ import { integValidateApiKeyController } from "../controllers/v0/admin/validate-
export const adminRouter = express.Router();
adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/redis-health`,
+ `/admin/${config.BULL_AUTH_KEY}/redis-health`,
redisHealthController,
);
adminRouter.post(
- `/admin/${process.env.BULL_AUTH_KEY}/acuc-cache-clear`,
+ `/admin/${config.BULL_AUTH_KEY}/acuc-cache-clear`,
wrap(acucCacheClearController),
);
adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/feng-check`,
+ `/admin/${config.BULL_AUTH_KEY}/feng-check`,
wrap(checkFireEngine),
);
-adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/cclog`,
- wrap(cclogController),
-);
+adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/cclog`, wrap(cclogController));
adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/zdrcleaner`,
+ `/admin/${config.BULL_AUTH_KEY}/zdrcleaner`,
wrap(zdrcleanerController),
);
adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/index-queue-prometheus`,
+ `/admin/${config.BULL_AUTH_KEY}/index-queue-prometheus`,
wrap(indexQueuePrometheus),
);
adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/precrawl`,
+ `/admin/${config.BULL_AUTH_KEY}/precrawl`,
wrap(triggerPrecrawl),
);
adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/metrics`,
+ `/admin/${config.BULL_AUTH_KEY}/metrics`,
wrap(metricsController),
);
adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/nuq-metrics`,
+ `/admin/${config.BULL_AUTH_KEY}/nuq-metrics`,
wrap(nuqMetricsController),
);
adminRouter.post(
- `/admin/${process.env.BULL_AUTH_KEY}/fsearch`,
+ `/admin/${config.BULL_AUTH_KEY}/fsearch`,
wrap(realtimeSearchController),
);
adminRouter.post(
- `/admin/${process.env.BULL_AUTH_KEY}/concurrency-queue-backfill`,
+ `/admin/${config.BULL_AUTH_KEY}/concurrency-queue-backfill`,
wrap(concurrencyQueueBackfillController),
);
diff --git a/apps/api/src/routes/domain-frequency.ts b/apps/api/src/routes/domain-frequency.ts
index e25fd7bce..2d7391c2a 100644
--- a/apps/api/src/routes/domain-frequency.ts
+++ b/apps/api/src/routes/domain-frequency.ts
@@ -1,4 +1,5 @@
import express from "express";
+import { config } from "../config";
import { Request, Response } from "express";
import {
getTopDomains,
@@ -12,7 +13,7 @@ const domainFrequencyRouter = express.Router();
// Get top domains by frequency
domainFrequencyRouter.get(
- `/domain-frequency/${process.env.BULL_AUTH_KEY}/top`,
+ `/domain-frequency/${config.BULL_AUTH_KEY}/top`,
async (req: Request, res: Response) => {
try {
const limit = parseInt(req.query.limit as string) || 100;
@@ -34,7 +35,7 @@ domainFrequencyRouter.get(
// Get frequency for a specific domain
domainFrequencyRouter.get(
- `/domain-frequency/${process.env.BULL_AUTH_KEY}/domain/:domain`,
+ `/domain-frequency/${config.BULL_AUTH_KEY}/domain/:domain`,
async (req: Request, res: Response) => {
try {
const { domain } = req.params;
@@ -70,7 +71,7 @@ domainFrequencyRouter.get(
// Get overall statistics
domainFrequencyRouter.get(
- `/domain-frequency/${process.env.BULL_AUTH_KEY}/stats`,
+ `/domain-frequency/${config.BULL_AUTH_KEY}/stats`,
async (req: Request, res: Response) => {
try {
const [stats, queueLength] = await Promise.all([
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index faa07fe1a..8e7dba30c 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -19,6 +19,7 @@ import * as geoip from "geoip-country";
import { isSelfHosted } from "../lib/deployment";
import { validate as isUuid } from "uuid";
+import { config } from "../config";
export function checkCreditsMiddleware(
_minimum?: number,
): (req: RequestWithAuth, res: Response, next: NextFunction) => void {
@@ -230,8 +231,7 @@ export function countryCheck(
return next();
}
- const restricted = process.env.RESTRICTED_COUNTRIES?.split(",") ?? [];
- if (restricted.includes(country.country)) {
+ if (config.RESTRICTED_COUNTRIES?.includes(country.country)) {
logger.warn("Denied access to restricted country", {
ip: req.ip,
country: country.country,
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index f4a20f896..c187ee3d1 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -1,4 +1,5 @@
import express from "express";
+import { config } from "../config";
import { crawlController } from "../controllers/v1/crawl";
// import { crawlStatusController } from "../../src/controllers/v1/crawl-status";
import { scrapeController } from "../../src/controllers/v1/scrape";
@@ -51,12 +52,12 @@ v1Router.use(requestTimingMiddleware("v1"));
// x402 payments protocol - https://github.com/coinbase/x402
// v1Router.use(
// paymentMiddleware(
-// (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||
+// (config.X402_PAY_TO_ADDRESS as `0x${string}`) ||
// "0x0000000000000000000000000000000000000000",
// {
// "POST /x402/search": {
-// price: process.env.X402_ENDPOINT_PRICE_USD as string,
-// network: process.env.X402_NETWORK as
+// price: config.X402_ENDPOINT_PRICE_USD as string,
+// network: config.X402_NETWORK as
// | "base-sepolia"
// | "base"
// | "avalanche-fuji"
@@ -311,12 +312,12 @@ v1Router.post(
authMiddleware(RateLimiterMode.Search),
countryCheck,
paymentMiddleware(
- (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||
+ (config.X402_PAY_TO_ADDRESS as `0x${string}`) ||
"0x0000000000000000000000000000000000000000",
{
"POST /x402/search": {
- price: process.env.X402_ENDPOINT_PRICE_USD as string,
- network: process.env.X402_NETWORK as
+ price: config.X402_ENDPOINT_PRICE_USD as string,
+ network: config.X402_NETWORK as
| "base-sepolia"
| "base"
| "avalanche-fuji"
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index f088fb632..68b39f45b 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -1,4 +1,5 @@
import express from "express";
+import { config } from "../config";
import { RateLimiterMode } from "../types";
import expressWs from "express-ws";
import { searchController } from "../controllers/v2/search";
@@ -48,12 +49,12 @@ v2Router.use(requestTimingMiddleware("v2"));
// x402 payments protocol - https://github.com/coinbase/x402
// v2Router.use(
// paymentMiddleware(
-// (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||
+// (config.X402_PAY_TO_ADDRESS as `0x${string}`) ||
// "0x0000000000000000000000000000000000000000",
// {
// "POST /x402/search": {
-// price: process.env.X402_ENDPOINT_PRICE_USD as string,
-// network: process.env.X402_NETWORK as
+// price: config.X402_ENDPOINT_PRICE_USD as string,
+// network: config.X402_NETWORK as
// | "base-sepolia"
// | "base"
// | "avalanche-fuji"
@@ -340,12 +341,12 @@ v2Router.post(
countryCheck,
blocklistMiddleware,
paymentMiddleware(
- (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||
+ (config.X402_PAY_TO_ADDRESS as `0x${string}`) ||
"0x0000000000000000000000000000000000000000",
{
"POST /x402/search": {
- price: process.env.X402_ENDPOINT_PRICE_USD as string,
- network: process.env.X402_NETWORK as
+ price: config.X402_ENDPOINT_PRICE_USD as string,
+ network: config.X402_NETWORK as
| "base-sepolia"
| "base"
| "avalanche-fuji"
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index e6e66b42a..410fe79d6 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -1,4 +1,5 @@
import { AxiosError } from "axios";
+import { config } from "../../config";
import { load } from "cheerio"; // rustified
import { URL } from "url";
import { getLinksFromSitemap } from "./sitemap";
@@ -186,7 +187,7 @@ export class WebCrawler {
fancyDenialReasons.set(key, DenialReason[value]);
});
- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
for (const link of res.links) {
this.logger.debug(`${link} OK`);
}
@@ -233,7 +234,7 @@ export class WebCrawler {
"file:",
];
if (nonWebProtocols.some(protocol => urlStr.startsWith(protocol))) {
- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} NON-WEB PROTOCOL FAIL`);
}
denialReasons.set(link, DenialReason.NON_WEB_PROTOCOL);
@@ -244,7 +245,7 @@ export class WebCrawler {
// Check if the link exceeds the maximum depth allowed
if (depth > maxDepth) {
- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} DEPTH FAIL`);
}
denialReasons.set(link, DenialReason.DEPTH_LIMIT);
@@ -260,7 +261,7 @@ export class WebCrawler {
new RegExp(excludePattern).test(excincPath),
)
) {
- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} EXCLUDE FAIL`);
}
denialReasons.set(link, DenialReason.EXCLUDE_PATTERN);
@@ -275,7 +276,7 @@ export class WebCrawler {
new RegExp(includePattern).test(excincPath),
)
) {
- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} INCLUDE FAIL`);
}
denialReasons.set(link, DenialReason.INCLUDE_PATTERN);
@@ -289,7 +290,7 @@ export class WebCrawler {
try {
normalizedLink = new URL(link);
} catch (_) {
- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} URL PARSE FAIL`);
}
return false;
@@ -310,7 +311,7 @@ export class WebCrawler {
if (
!normalizedLink.pathname.startsWith(normalizedInitialUrl.pathname)
) {
- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(
`${link} BACKWARDS FAIL ${normalizedLink.pathname} ${normalizedInitialUrl.pathname}`,
);
@@ -332,7 +333,7 @@ export class WebCrawler {
method: "filterLinks",
link,
});
- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} ROBOTS FAIL`);
}
denialReasons.set(link, DenialReason.ROBOTS_TXT);
@@ -340,14 +341,14 @@ export class WebCrawler {
}
if (this.isFile(link)) {
- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} FILE FAIL`);
}
denialReasons.set(link, DenialReason.FILE_TYPE);
return false;
}
- if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} OK`);
}
return true;
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 7c28a867e..d7faa57bd 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -1,4 +1,5 @@
import { parseStringPromise } from "xml2js";
+import { config } from "../../config";
import { WebCrawler, SITEMAP_LIMIT } from "./crawler";
import { scrapeURL } from "../scrapeURL";
import { scrapeOptions } from "../../controllers/v2/types";
@@ -19,8 +20,8 @@ import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";
import { useIndex } from "../../services";
const useFireEngine =
- process.env.FIRE_ENGINE_BETA_URL !== "" &&
- process.env.FIRE_ENGINE_BETA_URL !== undefined;
+ config.FIRE_ENGINE_BETA_URL !== "" &&
+ config.FIRE_ENGINE_BETA_URL !== undefined;
const gunzipAsync = promisify(gunzip);
diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/engine-forcing.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/engine-forcing.test.ts
index a0972f73f..f1f4469cd 100644
--- a/apps/api/src/scraper/WebScraper/utils/__tests__/engine-forcing.test.ts
+++ b/apps/api/src/scraper/WebScraper/utils/__tests__/engine-forcing.test.ts
@@ -1,8 +1,8 @@
import { initializeEngineForcing, getEngineForUrl } from "../engine-forcing";
-
+import { config } from "../../../../config";
describe("Engine Forcing", () => {
beforeEach(() => {
- delete process.env.FORCED_ENGINE_DOMAINS;
+ delete config.FORCED_ENGINE_DOMAINS;
});
describe("initializeEngineForcing", () => {
@@ -12,13 +12,13 @@ describe("Engine Forcing", () => {
});
it("should initialize with empty mappings when env var is empty", () => {
- process.env.FORCED_ENGINE_DOMAINS = "";
+ config.FORCED_ENGINE_DOMAINS = "";
initializeEngineForcing();
expect(getEngineForUrl("https://example.com")).toBeUndefined();
});
it("should parse valid JSON mappings", () => {
- process.env.FORCED_ENGINE_DOMAINS = JSON.stringify({
+ config.FORCED_ENGINE_DOMAINS = JSON.stringify({
"example.com": "playwright",
});
initializeEngineForcing();
@@ -26,7 +26,7 @@ describe("Engine Forcing", () => {
});
it("should handle invalid JSON gracefully", () => {
- process.env.FORCED_ENGINE_DOMAINS = "invalid json";
+ config.FORCED_ENGINE_DOMAINS = "invalid json";
initializeEngineForcing();
expect(getEngineForUrl("https://example.com")).toBeUndefined();
});
@@ -34,7 +34,7 @@ describe("Engine Forcing", () => {
describe("getEngineForUrl", () => {
beforeEach(() => {
- process.env.FORCED_ENGINE_DOMAINS = JSON.stringify({
+ config.FORCED_ENGINE_DOMAINS = JSON.stringify({
"example.com": "playwright",
"test.com": "fetch",
"*.subdomain.com": "fire-engine;chrome-cdp",
@@ -93,7 +93,7 @@ describe("Engine Forcing", () => {
});
it("should throw error if not initialized", () => {
- delete process.env.FORCED_ENGINE_DOMAINS;
+ delete config.FORCED_ENGINE_DOMAINS;
expect(() => {
const {
getEngineForUrl: freshGetEngineForUrl,
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index 1f7a3966d..f375f384c 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -1,4 +1,5 @@
import { configDotenv } from "dotenv";
+import { config } from "../../../config";
import { parse } from "tldts";
import { TeamFlags } from "../../../controllers/v1/types";
import { supabase_rr_service } from "../../../services/supabase";
@@ -13,10 +14,7 @@ type BlocklistBlob = {
let blob: BlocklistBlob | null = null;
export async function initializeBlocklist() {
- if (
- process.env.USE_DB_AUTHENTICATION !== "true" ||
- process.env.DISABLE_BLOCKLIST === "true"
- ) {
+ if (config.USE_DB_AUTHENTICATION !== true || config.DISABLE_BLOCKLIST) {
blob = {
blocklist: [],
allowedKeywords: [],
diff --git a/apps/api/src/scraper/WebScraper/utils/engine-forcing.ts b/apps/api/src/scraper/WebScraper/utils/engine-forcing.ts
index 89c972978..4aff39f66 100644
--- a/apps/api/src/scraper/WebScraper/utils/engine-forcing.ts
+++ b/apps/api/src/scraper/WebScraper/utils/engine-forcing.ts
@@ -3,6 +3,7 @@ import { parse } from "tldts";
import { Engine } from "../../scrapeURL/engines";
import { logger } from "../../../lib/logger";
+import { config } from "../../../config";
configDotenv();
type EngineForcingMapping = {
@@ -17,7 +18,7 @@ let engineMappings: EngineForcingMapping | null = null;
* Example: {"example.com": "playwright", "*.google.com": ["fire-engine;chrome-cdp", "playwright"]}
*/
export function initializeEngineForcing() {
- const envVar = process.env.FORCED_ENGINE_DOMAINS;
+ const envVar = config.FORCED_ENGINE_DOMAINS;
if (!envVar || envVar.trim() === "") {
engineMappings = {};
diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.ts
index c24bdc4ba..7319d9ce9 100644
--- a/apps/api/src/scraper/crawler/sitemap.ts
+++ b/apps/api/src/scraper/crawler/sitemap.ts
@@ -1,4 +1,5 @@
import type { Logger } from "winston";
+import { config } from "../../config";
import { scrapeOptions, ScrapeOptions } from "../../controllers/v2/types";
import { logger as _logger } from "../../lib/logger";
import { Engine } from "../scrapeURL/engines";
@@ -15,8 +16,8 @@ import { SitemapError } from "../../lib/error";
import { useIndex } from "../../services";
const useFireEngine =
- process.env.FIRE_ENGINE_BETA_URL !== "" &&
- process.env.FIRE_ENGINE_BETA_URL !== undefined;
+ config.FIRE_ENGINE_BETA_URL !== "" &&
+ config.FIRE_ENGINE_BETA_URL !== undefined;
type SitemapScrapeOptions = {
url: string;
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 7b5fd03b0..1d91851e8 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -19,6 +19,7 @@ import {
import { Meta } from "../..";
import { abTestFireEngine } from "../../../../services/ab-test";
+import { config } from "../../../../config";
export type FireEngineScrapeRequestCommon = {
url: string;
@@ -168,9 +169,9 @@ const failedSchema = z.object({
});
export const fireEngineURL =
- process.env.FIRE_ENGINE_BETA_URL ?? "<mock-fire-engine-url>";
+ config.FIRE_ENGINE_BETA_URL ?? "<mock-fire-engine-url>";
export const fireEngineStagingURL =
- process.env.FIRE_ENGINE_STAGING_URL ?? "<mock-fire-engine-url>";
+ config.FIRE_ENGINE_STAGING_URL ?? "<mock-fire-engine-url>";
export async function fireEngineScrape<
Engine extends
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 4e29065b2..0e54bdcec 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -1,4 +1,5 @@
import { ScrapeActionContent } from "../../../lib/entities";
+import { config } from "../../../config";
import { Meta } from "..";
import { documentMaxReasonableTime, scrapeDocument } from "./document";
import {
@@ -37,11 +38,11 @@ export type Engine =
| "index;documents";
const useFireEngine =
- process.env.FIRE_ENGINE_BETA_URL !== "" &&
- process.env.FIRE_ENGINE_BETA_URL !== undefined;
+ config.FIRE_ENGINE_BETA_URL !== "" &&
+ config.FIRE_ENGINE_BETA_URL !== undefined;
const usePlaywright =
- process.env.PLAYWRIGHT_MICROSERVICE_URL !== "" &&
- process.env.PLAYWRIGHT_MICROSERVICE_URL !== undefined;
+ config.PLAYWRIGHT_MICROSERVICE_URL !== "" &&
+ config.PLAYWRIGHT_MICROSERVICE_URL !== undefined;
const engines: Engine[] = [
...(useIndex ? ["index" as const, "index;documents" as const] : []),
@@ -465,7 +466,7 @@ const engineOptions: {
export function shouldUseIndex(meta: Meta) {
return (
useIndex &&
- process.env.FIRECRAWL_INDEX_WRITE_ONLY !== "true" &&
+ config.FIRECRAWL_INDEX_WRITE_ONLY !== true &&
!hasFormatOfType(meta.options.formats, "changeTracking") &&
!hasFormatOfType(meta.options.formats, "branding") &&
// Skip index if a non-default PDF maxPages is specified
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 7906fc9ac..06425d702 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -1,4 +1,5 @@
import crypto from "crypto";
+import { config } from "../../../../config";
import { Document } from "../../../../controllers/v1/types";
import { EngineScrapeResult } from "..";
import { Meta } from "../..";
@@ -208,8 +209,8 @@ export async function scrapeURLWithIndex(
if (
domainSplitsHash.length === 0 ||
- process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true" ||
- process.env.USE_DB_AUTHENTICATION !== "true"
+ config.FIRECRAWL_INDEX_WRITE_ONLY ||
+ config.USE_DB_AUTHENTICATION !== true
) {
maxAge = 2 * 24 * 60 * 60 * 1000; // 2 days
} else {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 07e5f4146..b8420e8aa 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -1,4 +1,5 @@
import { Meta } from "../..";
+import { config } from "../../../../config";
import { EngineScrapeResult } from "..";
import * as marked from "marked";
import { robustFetch } from "../../lib/fetch";
@@ -68,10 +69,9 @@ async function scrapePDFWithRunPodMU(
});
if (
- process.env.PDF_MU_V2_EXPERIMENT === "true" &&
- process.env.PDF_MU_V2_BASE_URL &&
- Math.random() * 100 <
- Number(process.env.PDF_MU_V2_EXPERIMENT_PERCENT ?? "100")
+ config.PDF_MU_V2_EXPERIMENT === "true" &&
+ config.PDF_MU_V2_BASE_URL &&
+ Math.random() * 100 < config.PDF_MU_V2_EXPERIMENT_PERCENT
) {
(async () => {
const pdfParseId = crypto.randomUUID();
@@ -85,9 +85,11 @@ async function scrapePDFWithRunPodMU(
});
try {
const resp = await robustFetch({
- url: process.env.PDF_MU_V2_BASE_URL ?? "",
+ url: config.PDF_MU_V2_BASE_URL ?? "",
method: "POST",
- headers: process.env.PDF_MU_V2_API_KEY ? { Authorization: `Bearer ${process.env.PDF_MU_V2_API_KEY}` } : undefined,
+ headers: config.PDF_MU_V2_API_KEY
+ ? { Authorization: `Bearer ${config.PDF_MU_V2_API_KEY}` }
+ : undefined,
body: {
input: {
file_content: base64Content,
@@ -124,11 +126,10 @@ async function scrapePDFWithRunPodMU(
const muV1StartedAt = Date.now();
const podStart = await robustFetch({
- url:
- "https://api.runpod.ai/v2/" + process.env.RUNPOD_MU_POD_ID + "/runsync",
+ url: "https://api.runpod.ai/v2/" + config.RUNPOD_MU_POD_ID + "/runsync",
method: "POST",
headers: {
- Authorization: `Bearer ${process.env.RUNPOD_MU_API_KEY}`,
+ Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,
},
body: {
input: {
@@ -164,10 +165,10 @@ async function scrapePDFWithRunPodMU(
await new Promise(resolve => setTimeout(resolve, 2500));
meta.abort.throwIfAborted();
const podStatus = await robustFetch({
- url: `https://api.runpod.ai/v2/${process.env.RUNPOD_MU_POD_ID}/status/${podStart.id}`,
+ url: `https://api.runpod.ai/v2/${config.RUNPOD_MU_POD_ID}/status/${podStart.id}`,
method: "GET",
headers: {
- Authorization: `Bearer ${process.env.RUNPOD_MU_API_KEY}`,
+ Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,
},
logger: meta.logger.child({
method: "scrapePDFWithRunPodMU/status/robustFetch",
@@ -359,8 +360,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
// First try RunPod MU if conditions are met
if (
base64Content.length < MAX_FILE_SIZE &&
- process.env.RUNPOD_MU_API_KEY &&
- process.env.RUNPOD_MU_POD_ID
+ config.RUNPOD_MU_API_KEY &&
+ config.RUNPOD_MU_POD_ID
) {
const muV1StartedAt = Date.now();
try {
diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
index ae2bfec97..1acb185b9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
@@ -1,4 +1,5 @@
import { z } from "zod";
+import { config } from "../../../../config";
import { EngineScrapeResult } from "..";
import { Meta } from "../..";
import { robustFetch } from "../../lib/fetch";
@@ -8,7 +9,7 @@ export async function scrapeURLWithPlaywright(
meta: Meta,
): Promise<EngineScrapeResult> {
const response = await robustFetch({
- url: process.env.PLAYWRIGHT_MICROSERVICE_URL!,
+ url: config.PLAYWRIGHT_MICROSERVICE_URL!,
headers: {
"Content-Type": "application/json",
},
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
index a430e50bb..597560258 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
@@ -1,10 +1,10 @@
import type { Socket } from "net";
+import { config } from "../../../../config";
import type { TLSSocket } from "tls";
import * as undici from "undici";
import { CookieJar } from "tough-cookie";
import { cookie } from "http-cookie-agent/undici";
import IPAddr from "ipaddr.js";
-
export class InsecureConnectionError extends Error {
constructor() {
super("Connection violated security rules.");
@@ -23,13 +23,13 @@ function createBaseAgent(skipTlsVerification: boolean) {
maxRedirections: 5000,
};
- return process.env.PROXY_SERVER
+ return config.PROXY_SERVER
? new undici.ProxyAgent({
- uri: process.env.PROXY_SERVER.includes("://")
- ? process.env.PROXY_SERVER
- : "http://" + process.env.PROXY_SERVER,
- token: process.env.PROXY_USERNAME
- ? `Basic ${Buffer.from(process.env.PROXY_USERNAME + ":" + (process.env.PROXY_PASSWORD ?? "")).toString("base64")}`
+ uri: config.PROXY_SERVER.includes("://")
+ ? config.PROXY_SERVER
+ : "http://" + config.PROXY_SERVER,
+ token: config.PROXY_USERNAME
+ ? `Basic ${Buffer.from(config.PROXY_USERNAME + ":" + (config.PROXY_PASSWORD ?? "")).toString("base64")}`
: undefined,
requestTls: {
rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested
@@ -55,7 +55,7 @@ function attachSecurityCheck(agent: undici.Dispatcher) {
if (
socket.remoteAddress &&
isIPPrivate(socket.remoteAddress) &&
- process.env.ALLOW_LOCAL_WEBHOOKS !== "true"
+ config.ALLOW_LOCAL_WEBHOOKS !== true
) {
socket.destroy(new InsecureConnectionError());
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 0126b2e4b..9305a2c23 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1,4 +1,5 @@
import { Logger } from "winston";
+import { config } from "../../config";
import * as Sentry from "@sentry/node";
import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
@@ -499,7 +500,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
const waitUntilWaterfall =
getEngineMaxReasonableTime(meta, engine) +
- parseInt(process.env.SCRAPEURL_ENGINE_WATERFALL_DELAY_MS || "0", 10);
+ config.SCRAPEURL_ENGINE_WATERFALL_DELAY_MS;
if (
!isFinite(waitUntilWaterfall) ||
@@ -923,7 +924,7 @@ export async function scrapeURL(
useIndex &&
meta.options.storeInCache &&
!meta.internalOptions.zeroDataRetention &&
- internalOptions.teamId !== process.env.PRECRAWL_TEAM_ID &&
+ internalOptions.teamId !== config.PRECRAWL_TEAM_ID &&
meta.internalOptions.isPreCrawl !== true; // sitemap crawls override teamId but keep the isPreCrawl flag
if (shouldRecordFrequency) {
(async () => {
diff --git a/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts b/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts
index dd7c6a0a5..a08076875 100644
--- a/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts
@@ -1,7 +1,8 @@
import CacheableLookup from "cacheable-lookup";
import dns from "dns";
+import { config } from "../../../config";
export const cacheableLookup =
- process.env.SENTRY_ENVIRONMENT === "dev"
+ config.SENTRY_ENVIRONMENT === "dev"
? { lookup: dns.lookup, install: () => {} }
: new CacheableLookup({});
diff --git a/apps/api/src/scraper/scrapeURL/lib/mock.ts b/apps/api/src/scraper/scrapeURL/lib/mock.ts
index d01dc690e..168b9b35d 100644
--- a/apps/api/src/scraper/scrapeURL/lib/mock.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/mock.ts
@@ -1,4 +1,5 @@
import * as fs from "fs/promises";
+import { config } from "../../../config";
import * as path from "path";
import { logger as _logger } from "../../../lib/logger";
import { Logger } from "winston";
@@ -8,7 +9,7 @@ const loadMocksDirPath = path
.replace("dist/", "");
export async function saveMock(options: unknown, result: unknown) {
- if (process.env.FIRECRAWL_SAVE_MOCKS !== "true") return;
+ if (config.FIRECRAWL_SAVE_MOCKS !== true) return;
await fs.mkdir(saveMocksDirPath, { recursive: true });
diff --git a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
index 93decdaf2..8c9304881 100644
--- a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
@@ -1,4 +1,5 @@
import { z } from "zod";
+import { config } from "../../../config";
import { logger as _logger } from "../../../lib/logger";
import { robustFetch } from "./fetch";
import fs from "fs/promises";
@@ -79,7 +80,7 @@ export async function smartScrape({
// Pass schema type as generic parameter to robustFeth
const response = await robustFetch<typeof smartScrapeResultSchema>({
- url: `${process.env.SMART_SCRAPE_API_URL}/smart-scrape`,
+ url: `${config.SMART_SCRAPE_API_URL}/smart-scrape`,
method: "POST",
body: {
url,
diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
index e8336d999..f183df28e 100644
--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
@@ -1,6 +1,7 @@
import "dotenv/config";
-process.env.ENV = "test";
+import { config } from "../../config";
+config.ENV = "test";
import { scrapeURL } from ".";
import { scrapeOptions } from "../../controllers/v2/types";
diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
index d630a5f28..01dc391b0 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
@@ -10,6 +10,7 @@
*/
import { Document } from "../../../controllers/v1/types";
+import { config } from "../../../config";
import { indexDocumentIfEnabled } from "../../../lib/search-index-client";
import { logger as _logger } from "../../../lib/logger";
import { Meta } from "..";
@@ -55,13 +56,12 @@ function shouldIndexForSearch(meta: Meta, document: Document): boolean {
*/
function shouldSampleDocument(): boolean {
// Get sample rate from environment (default 10% for safe rollout)
- const sampleRateStr = process.env.SEARCH_INDEX_SAMPLE_RATE || "0.1";
- const sampleRate = parseFloat(sampleRateStr);
+ const sampleRate = config.SEARCH_INDEX_SAMPLE_RATE;
// Validate sample rate
if (isNaN(sampleRate) || sampleRate < 0 || sampleRate > 1) {
_logger.warn("Invalid SEARCH_INDEX_SAMPLE_RATE, using 0.1 (10%)", {
- value: sampleRateStr,
+ value: sampleRate,
});
return Math.random() < 0.1;
}
@@ -79,8 +79,7 @@ export async function sendDocumentToSearchIndex(
): Promise<Document> {
// Check if search indexing is enabled via the SEARCH_SERVICE_URL
const searchIndexEnabled =
- process.env.ENABLE_SEARCH_INDEX === "true" &&
- process.env.SEARCH_SERVICE_URL;
+ config.ENABLE_SEARCH_INDEX && config.SEARCH_SERVICE_URL;
meta.logger.debug("Sending document to search index", {
url: meta.url,
@@ -95,7 +94,7 @@ export async function sendDocumentToSearchIndex(
if (!shouldSampleDocument()) {
meta.logger.debug("Document not sampled for search indexing", {
url: meta.url,
- sampleRate: process.env.SEARCH_INDEX_SAMPLE_RATE || "0.1",
+ sampleRate: config.SEARCH_INDEX_SAMPLE_RATE,
});
return document;
}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts b/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts
index 83df17b84..55cb5c9f8 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts
@@ -2,12 +2,13 @@
// and it makes me sad. - mogery
import { supabase_service } from "../../../services/supabase";
+import { config } from "../../../config";
import { Meta } from "..";
import { Document } from "../../../controllers/v1/types";
export function uploadScreenshot(meta: Meta, document: Document): Document {
if (
- process.env.USE_DB_AUTHENTICATION === "true" &&
+ config.USE_DB_AUTHENTICATION &&
document.screenshot !== undefined &&
document.screenshot.startsWith("data:")
) {
diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.ts
index f7264756d..8d3f55cf4 100644
--- a/apps/api/src/search/fireEngine.ts
+++ b/apps/api/src/search/fireEngine.ts
@@ -1,14 +1,12 @@
-import dotenv from "dotenv";
+import { config } from "../config";
import { SearchResult } from "../../src/lib/entities";
import * as Sentry from "@sentry/node";
import { logger } from "../lib/logger";
import { executeWithRetry, attemptRequest } from "../lib/retry-utils";
-dotenv.config();
-
const useFireEngine =
- process.env.FIRE_ENGINE_BETA_URL !== "" &&
- process.env.FIRE_ENGINE_BETA_URL !== undefined;
+ config.FIRE_ENGINE_BETA_URL !== "" &&
+ config.FIRE_ENGINE_BETA_URL !== undefined;
function hasResults(results: unknown): results is SearchResult[] {
return Array.isArray(results) && results.length > 0;
@@ -41,7 +39,7 @@ export async function fire_engine_search(
page: options.page ?? 1,
};
- const url = `${process.env.FIRE_ENGINE_BETA_URL}/search`;
+ const url = `${config.FIRE_ENGINE_BETA_URL}/search`;
const data = JSON.stringify(payload);
const result = await executeWithRetry<SearchResult[]>(
@@ -83,7 +81,7 @@ export async function fireEngineMap(
page: options.page ?? 1,
};
- const url = `${process.env.FIRE_ENGINE_BETA_URL}/map`;
+ const url = `${config.FIRE_ENGINE_BETA_URL}/map`;
const data = JSON.stringify(payload);
const result = await executeWithRetry<SearchResult[]>(
diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.ts
index 40fd00d70..027724329 100644
--- a/apps/api/src/search/index.ts
+++ b/apps/api/src/search/index.ts
@@ -1,4 +1,5 @@
import { SearchResult } from "../../src/lib/entities";
+import { config } from "../config";
import { searxng_search } from "./searxng";
import { fire_engine_search } from "./fireEngine";
import { Logger } from "winston";
@@ -32,7 +33,7 @@ export async function search({
timeout?: number;
}): Promise<SearchResult[]> {
try {
- if (process.env.FIRE_ENGINE_BETA_URL) {
+ if (config.FIRE_ENGINE_BETA_URL) {
logger.info("Using fire engine search");
const results = await fire_engine_search(query, {
numResults: num_results,
@@ -44,7 +45,7 @@ export async function search({
});
return results;
}
- if (process.env.SEARXNG_ENDPOINT) {
+ if (config.SEARXNG_ENDPOINT) {
logger.info("Using searxng search");
const results = await searxng_search(query, {
num_results,
diff --git a/apps/api/src/search/searxng.ts b/apps/api/src/search/searxng.ts
index 11d5a9e32..46017dfa0 100644
--- a/apps/api/src/search/searxng.ts
+++ b/apps/api/src/search/searxng.ts
@@ -1,10 +1,8 @@
import axios from "axios";
-import dotenv from "dotenv";
+import { config } from "../config";
import { SearchResult } from "../../src/lib/entities";
import { logger } from "../lib/logger";
-dotenv.config();
-
interface SearchOptions {
tbs?: string;
filter?: string;
@@ -25,13 +23,13 @@ export async function searxng_search(
// gl: options.country, //not possible with SearXNG
// location: options.location, //not possible with SearXNG
// num: options.num_results, //not possible with SearXNG
- engines: process.env.SEARXNG_ENGINES || "",
- categories: process.env.SEARXNG_CATEGORIES || "",
+ engines: config.SEARXNG_ENGINES ?? "",
+ categories: config.SEARXNG_CATEGORIES ?? "",
pageno: options.page ?? 1,
format: "json",
};
- const url = process.env.SEARXNG_ENDPOINT!;
+ const url = config.SEARXNG_ENDPOINT!;
// Remove trailing slash if it exists
const cleanedUrl = url.endsWith("/") ? url.slice(0, -1) : url;
diff --git a/apps/api/src/search/v2/ddgsearch.ts b/apps/api/src/search/v2/ddgsearch.ts
index 2f05cbb14..9838423ac 100644
--- a/apps/api/src/search/v2/ddgsearch.ts
+++ b/apps/api/src/search/v2/ddgsearch.ts
@@ -1,4 +1,5 @@
import * as undici from "undici";
+import { config } from "../../config";
import { JSDOM } from "jsdom";
import { SearchV2Response, WebSearchResult } from "../../lib/entities";
import { logger } from "../../lib/logger";
@@ -224,7 +225,7 @@ export async function ddgSearch(
return { web: results.slice(0, num_results) };
} catch (error: any) {
if (error instanceof DDGAntiBotError) {
- if (process.env.TEST_SUITE_SELF_HOSTED) {
+ if (config.TEST_SUITE_SELF_HOSTED) {
logger.warn(
"DuckDuckGo: Blocked by anti-bot measures, returning dummy page for test suite...",
{ term },
diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.ts
index e89f53531..9ba5b1d86 100644
--- a/apps/api/src/search/v2/fireEngine-v2.ts
+++ b/apps/api/src/search/v2/fireEngine-v2.ts
@@ -1,4 +1,4 @@
-import dotenv from "dotenv";
+import { config } from "../../config";
import {
SearchResult,
SearchV2Response,
@@ -8,11 +8,9 @@ import * as Sentry from "@sentry/node";
import { logger } from "../../lib/logger";
import { executeWithRetry, attemptRequest } from "../../lib/retry-utils";
-dotenv.config();
-
const useFireEngine =
- process.env.FIRE_ENGINE_BETA_URL !== "" &&
- process.env.FIRE_ENGINE_BETA_URL !== undefined;
+ config.FIRE_ENGINE_BETA_URL !== "" &&
+ config.FIRE_ENGINE_BETA_URL !== undefined;
function normalizeSearchTypes(
type?: SearchResultType | SearchResultType[],
@@ -72,7 +70,7 @@ export async function fire_engine_search_v2(
};
const requestedTypes = normalizeSearchTypes(options.type);
- const url = `${process.env.FIRE_ENGINE_BETA_URL}/v2/search`;
+ const url = `${config.FIRE_ENGINE_BETA_URL}/v2/search`;
const data = JSON.stringify(payload);
const result = await executeWithRetry<SearchV2Response>(
diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.ts
index 112ba5d09..3dca1a848 100644
--- a/apps/api/src/search/v2/index.ts
+++ b/apps/api/src/search/v2/index.ts
@@ -1,4 +1,5 @@
import { SearchV2Response, SearchResultType } from "../../lib/entities";
+import { config } from "../../config";
import { fire_engine_search_v2 } from "./fireEngine-v2";
import { searxng_search } from "./searxng";
import { ddgSearch } from "./ddgsearch";
@@ -36,7 +37,7 @@ export async function search({
enterprise?: ("default" | "anon" | "zdr")[];
}): Promise<SearchV2Response> {
try {
- if (process.env.FIRE_ENGINE_BETA_URL) {
+ if (config.FIRE_ENGINE_BETA_URL) {
logger.info("Using fire engine search");
const results = await fire_engine_search_v2(query, {
numResults: num_results,
@@ -52,7 +53,7 @@ export async function search({
return results;
}
- if (process.env.SEARXNG_ENDPOINT) {
+ if (config.SEARXNG_ENDPOINT) {
logger.info("Using searxng search");
const results = await searxng_search(query, {
num_results,
diff --git a/apps/api/src/search/v2/searxng.ts b/apps/api/src/search/v2/searxng.ts
index 550a35aec..99af9f9ea 100644
--- a/apps/api/src/search/v2/searxng.ts
+++ b/apps/api/src/search/v2/searxng.ts
@@ -1,10 +1,8 @@
import axios from "axios";
-import dotenv from "dotenv";
+import { config } from "../../config";
import { SearchV2Response, WebSearchResult } from "../../lib/entities";
import { logger } from "../../lib/logger";
-dotenv.config();
-
interface SearchOptions {
tbs?: string;
filter?: string;
@@ -25,13 +23,13 @@ export async function searxng_search(
// gl: options.country, //not possible with SearXNG
// location: options.location, //not possible with SearXNG
// num: options.num_results, //not possible with SearXNG
- engines: process.env.SEARXNG_ENGINES || "",
- categories: process.env.SEARXNG_CATEGORIES || "",
+ engines: config.SEARXNG_ENGINES ?? "",
+ categories: config.SEARXNG_CATEGORIES ?? "",
pageno: options.page ?? 1,
format: "json",
};
- const url = process.env.SEARXNG_ENDPOINT!;
+ const url = config.SEARXNG_ENDPOINT!;
// Remove trailing slash if it exists
const cleanedUrl = url.endsWith("/") ? url.slice(0, -1) : url;
diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.ts
index 9be7a6057..a9ac8f19f 100644
--- a/apps/api/src/services/ab-test.ts
+++ b/apps/api/src/services/ab-test.ts
@@ -1,6 +1,7 @@
import { ScrapeJobData } from "../types";
import { logger as _logger } from "../lib/logger";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
+import { config } from "../config";
import {
FireEngineScrapeRequestChromeCDP,
FireEngineScrapeRequestCommon,
@@ -12,10 +13,9 @@ export function abTestJob(webScraperOptions: ScrapeJobData) {
// Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE
const abLogger = _logger.child({ method: "ABTestToStaging" });
try {
- const abRateEnv = process.env.SCRAPEURL_AB_RATE;
- const abHostEnv = process.env.SCRAPEURL_AB_HOST;
- const shouldExtendMaxAge =
- process.env.SCRAPEURL_AB_EXTEND_MAXAGE === "true";
+ const abRateEnv = config.SCRAPEURL_AB_RATE;
+ const abHostEnv = config.SCRAPEURL_AB_HOST;
+ const shouldExtendMaxAge = config.SCRAPEURL_AB_EXTEND_MAXAGE;
const abRate =
abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
const shouldABTest =
@@ -82,8 +82,8 @@ export function abTestFireEngine(
// Global A/B test: mirror request to staging fire-engine based on SCRAPEURL_AB_RATE
const abLogger = _logger.child({ method: "ABTestToStaging" });
try {
- const abRateEnv = process.env.FIRE_ENGINE_AB_RATE;
- const abHostEnv = process.env.FIRE_ENGINE_AB_HOST;
+ const abRateEnv = config.FIRE_ENGINE_AB_RATE;
+ const abHostEnv = config.FIRE_ENGINE_AB_HOST;
const abRate =
abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
const shouldABTest =
diff --git a/apps/api/src/services/agentLivecastWS.ts b/apps/api/src/services/agentLivecastWS.ts
index 4964ac83d..32af19d86 100644
--- a/apps/api/src/services/agentLivecastWS.ts
+++ b/apps/api/src/services/agentLivecastWS.ts
@@ -1,4 +1,5 @@
import { configDotenv } from "dotenv";
+import { config } from "../config";
import { logger } from "../lib/logger";
import type { Request } from "express";
import WSWebSocket from "ws";
@@ -26,7 +27,7 @@ export function attachWsProxy(app: any) {
const url = new URL(req.url ?? "", "http://placeholder/");
const sessionIdParam = url.searchParams.get("userProvidedId") || "";
- const workerWsUrl = `${process.env.FIRE_ENGINE_BETA_URL?.replace("http", "ws")}?userProvidedId=${sessionIdParam}`;
+ const workerWsUrl = `${config.FIRE_ENGINE_BETA_URL?.replace("http", "ws")}?userProvidedId=${sessionIdParam}`;
console.log(workerWsUrl);
const wsWorker = new WebSocket(workerWsUrl);
diff --git a/apps/api/src/services/alerts/slack.ts b/apps/api/src/services/alerts/slack.ts
index ad8f9186a..cfee2aafd 100644
--- a/apps/api/src/services/alerts/slack.ts
+++ b/apps/api/src/services/alerts/slack.ts
@@ -1,10 +1,11 @@
import axios from "axios";
import { logger } from "../../../src/lib/logger";
+import { config } from "../../config";
export async function sendSlackWebhook(
message: string,
alertEveryone: boolean = false,
- webhookUrl: string = process.env.SLACK_WEBHOOK_URL ?? "",
+ webhookUrl: string = config.SLACK_WEBHOOK_URL ?? "",
) {
const messagePrefix = alertEveryone ? "<!channel> " : "";
const payload = {
diff --git a/apps/api/src/services/billing/auto_charge.ts b/apps/api/src/services/billing/auto_charge.ts
index 57ca143d7..735871006 100644
--- a/apps/api/src/services/billing/auto_charge.ts
+++ b/apps/api/src/services/billing/auto_charge.ts
@@ -1,5 +1,6 @@
// Import necessary dependencies and types
import { AuthCreditUsageChunk } from "../../controllers/v1/types";
+import { config } from "../../config";
import { clearACUC, clearACUCTeam, getACUC } from "../../controllers/auth";
import { redlock } from "../redlock";
import { supabase_rr_service, supabase_service } from "../supabase";
@@ -332,11 +333,11 @@ async function _autoChargeScale(
logger.info("Scale auto-recharge successful");
- if (process.env.SLACK_ADMIN_WEBHOOK_URL) {
+ if (config.SLACK_ADMIN_WEBHOOK_URL) {
sendSlackWebhook(
`💰 Auto-recharge successful on team ${chunk.team_id} for ${price.credits} credits (total auto-recharges this month: ${rechargesThisMonth.length + 1}).`,
false,
- process.env.SLACK_ADMIN_WEBHOOK_URL,
+ config.SLACK_ADMIN_WEBHOOK_URL,
).catch(error => {
logger.debug(`Error sending slack notification: ${error}`);
});
@@ -580,7 +581,7 @@ async function _autoChargeSelfServe(
paymentStatus: paymentStatus.return_status,
});
- if (process.env.SLACK_ADMIN_WEBHOOK_URL) {
+ if (config.SLACK_ADMIN_WEBHOOK_URL) {
const webhookCooldownKey = `webhook_cooldown_${chunk.team_id}`;
const isInCooldown = await getValue(webhookCooldownKey);
@@ -588,7 +589,7 @@ async function _autoChargeSelfServe(
sendSlackWebhook(
`Auto-recharge: Team ${chunk.team_id}. ${AUTO_RECHARGE_CREDITS} credits added. Payment status: ${paymentStatus.return_status}.`,
false,
- process.env.SLACK_ADMIN_WEBHOOK_URL,
+ config.SLACK_ADMIN_WEBHOOK_URL,
).catch(error => {
logger.debug(
`Error sending slack notification: ${error}`,
diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.ts
index b73d2b67a..8878c2145 100644
--- a/apps/api/src/services/billing/batch_billing.ts
+++ b/apps/api/src/services/billing/batch_billing.ts
@@ -1,4 +1,5 @@
import { logger } from "../../lib/logger";
+import { config } from "../../config";
import { getRedisConnection } from "../queue-service";
import { supabase_service } from "../supabase";
import * as Sentry from "@sentry/node";
@@ -241,7 +242,7 @@ export async function queueBillingOperation(
// Should we add this?
// I guess batch is fast enough that it's fine
- // if (process.env.USE_DB_AUTHENTICATION === "true") {
+ // if (config.USE_DB_AUTHENTICATION) {
// (async () => {
// // Get API keys for this team to update in cache
// const { data } = await supabase_service
diff --git a/apps/api/src/services/billing/stripe.ts b/apps/api/src/services/billing/stripe.ts
index 8a43e53ca..4e3444da0 100644
--- a/apps/api/src/services/billing/stripe.ts
+++ b/apps/api/src/services/billing/stripe.ts
@@ -1,7 +1,8 @@
import { logger } from "../../lib/logger";
+import { config } from "../../config";
import Stripe from "stripe";
-const stripe = new Stripe(process.env.STRIPE_SECRET_KEY ?? "");
+const stripe = new Stripe(config.STRIPE_SECRET_KEY ?? "");
async function getCustomerDefaultPaymentMethod(customerId: string) {
const paymentMethods = await stripe.customers.listPaymentMethods(customerId, {
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index f57739344..007705d22 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -1,4 +1,5 @@
import "dotenv/config";
+import { config } from "../config";
import "./sentry";
import { setSentryServiceTag } from "./sentry";
import * as Sentry from "@sentry/node";
@@ -26,16 +27,12 @@ configDotenv();
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
-const jobLockExtendInterval =
- Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;
-const jobLockExtensionTime =
- Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;
+const jobLockExtendInterval = config.JOB_LOCK_EXTEND_INTERVAL;
+const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;
-const cantAcceptConnectionInterval =
- Number(process.env.CANT_ACCEPT_CONNECTION_INTERVAL) || 2000;
-const connectionMonitorInterval =
- Number(process.env.CONNECTION_MONITOR_INTERVAL) || 10;
-const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;
+const cantAcceptConnectionInterval = config.CANT_ACCEPT_CONNECTION_INTERVAL;
+const connectionMonitorInterval = config.CONNECTION_MONITOR_INTERVAL;
+const gotJobInterval = config.CONNECTION_MONITOR_INTERVAL;
const runningJobs: Set<string> = new Set();
@@ -259,11 +256,11 @@ let currentLiveness: boolean = true;
app.get("/liveness", (req, res) => {
_logger.info("Liveness endpoint hit");
- if (process.env.USE_DB_AUTHENTICATION === "true") {
+ if (config.USE_DB_AUTHENTICATION) {
// networking check for Kubernetes environments
- const host = process.env.FIRECRAWL_APP_HOST || "firecrawl-app-service";
- const port = process.env.FIRECRAWL_APP_PORT || "3002";
- const scheme = process.env.FIRECRAWL_APP_SCHEME || "http";
+ const host = config.FIRECRAWL_APP_HOST;
+ const port = config.FIRECRAWL_APP_PORT;
+ const scheme = config.FIRECRAWL_APP_SCHEME;
robustFetch({
url: `${scheme}://${host}:${port}`,
@@ -289,7 +286,7 @@ app.get("/liveness", (req, res) => {
}
});
-const workerPort = process.env.EXTRACT_WORKER_PORT || process.env.PORT || 3005;
+const workerPort = config.EXTRACT_WORKER_PORT || config.PORT;
app.listen(workerPort, () => {
_logger.info(`Liveness endpoint is running on port ${workerPort}`);
});
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index c262dfc99..4d4f54cc7 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -10,6 +10,7 @@ import { MapDocument } from "../controllers/v2/types";
import { PdfMetadata } from "@mendable/firecrawl-rs";
import { storage } from "../lib/gcs-jobs";
import { withSpan, setSpanAttributes } from "../lib/otel-tracer";
+import { config } from "../config";
configDotenv();
// SupabaseService class initializes the Supabase client conditionally based on environment variables.
@@ -17,8 +18,8 @@ class IndexSupabaseService {
private client: SupabaseClient | null = null;
constructor() {
- const supabaseUrl = process.env.INDEX_SUPABASE_URL;
- const supabaseServiceToken = process.env.INDEX_SUPABASE_SERVICE_TOKEN;
+ const supabaseUrl = config.INDEX_SUPABASE_URL;
+ const supabaseServiceToken = config.INDEX_SUPABASE_SERVICE_TOKEN;
// Only initialize the Supabase client if both URL and Service Token are provided.
if (!supabaseUrl || !supabaseServiceToken) {
// Warn the user that Authentication is disabled by setting the client to null
@@ -57,8 +58,8 @@ export const index_supabase_service: SupabaseClient = new Proxy(serv, {
},
}) as unknown as SupabaseClient;
-const credentials = process.env.GCS_CREDENTIALS
- ? JSON.parse(atob(process.env.GCS_CREDENTIALS))
+const credentials = config.GCS_CREDENTIALS
+ ? JSON.parse(atob(config.GCS_CREDENTIALS))
: undefined;
export async function getIndexFromGCS(
@@ -72,12 +73,12 @@ export async function getIndexFromGCS(
"index.url": url,
});
- if (!process.env.GCS_INDEX_BUCKET_NAME) {
+ if (!config.GCS_INDEX_BUCKET_NAME) {
setSpanAttributes(span, { "gcs.index_bucket_configured": false });
return null;
}
- const bucket = storage.bucket(process.env.GCS_INDEX_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_INDEX_BUCKET_NAME);
const blob = bucket.file(`${url}`);
const [blobContent] = await blob.download();
const parsed = JSON.parse(blobContent.toString());
@@ -106,7 +107,7 @@ export async function getIndexFromGCS(
) {
logger?.info("Re-signing screenshot URL");
const [url] = await storage
- .bucket(process.env.GCS_MEDIA_BUCKET_NAME!)
+ .bucket(config.GCS_MEDIA_BUCKET_NAME!)
.file(decodeURIComponent(screenshotUrl.pathname.split("/")[2]))
.getSignedUrl({
action: "read",
@@ -169,12 +170,12 @@ export async function saveIndexToGCS(
"index.has_error": !!doc.error,
});
- if (!process.env.GCS_INDEX_BUCKET_NAME) {
+ if (!config.GCS_INDEX_BUCKET_NAME) {
setSpanAttributes(span, { "gcs.index_bucket_configured": false });
return;
}
- const bucket = storage.bucket(process.env.GCS_INDEX_BUCKET_NAME);
+ const bucket = storage.bucket(config.GCS_INDEX_BUCKET_NAME);
const blob = bucket.file(`${id}.json`);
for (let i = 0; i < 3; i++) {
@@ -200,12 +201,11 @@ export async function saveIndexToGCS(
}
export const useIndex =
- process.env.INDEX_SUPABASE_URL !== "" &&
- process.env.INDEX_SUPABASE_URL !== undefined;
+ config.INDEX_SUPABASE_URL !== "" && config.INDEX_SUPABASE_URL !== undefined;
export const useSearchIndex =
- process.env.SEARCH_INDEX_SUPABASE_URL !== "" &&
- process.env.SEARCH_INDEX_SUPABASE_URL !== undefined;
+ config.SEARCH_INDEX_SUPABASE_URL !== "" &&
+ config.SEARCH_INDEX_SUPABASE_URL !== undefined;
export function normalizeURLForIndex(url: string): string {
const urlObj = new URL(url);
@@ -664,7 +664,7 @@ export async function queryIndexAtSplitLevel(
limit: number,
maxAge = 2 * 24 * 60 * 60 * 1000,
): Promise<string[]> {
- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {
return [];
}
@@ -717,7 +717,7 @@ export async function queryIndexAtDomainSplitLevel(
limit: number,
maxAge = 2 * 24 * 60 * 60 * 1000,
): Promise<string[]> {
- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {
return [];
}
@@ -769,7 +769,7 @@ export async function queryOMCESignatures(
hostname: string,
maxAge = 2 * 24 * 60 * 60 * 1000,
): Promise<string[]> {
- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {
return [];
}
@@ -800,7 +800,7 @@ export async function queryIndexAtSplitLevelWithMeta(
url: string,
limit: number,
): Promise<MapDocument[]> {
- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {
return [];
}
@@ -860,7 +860,7 @@ export async function queryIndexAtDomainSplitLevelWithMeta(
hostname: string,
limit: number,
): Promise<MapDocument[]> {
- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {
return [];
}
@@ -928,7 +928,7 @@ export async function queryDomainsForPrecrawl(
maxDomains = 50,
logger: Logger = _logger,
): Promise<DomainPriority[]> {
- if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
+ if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {
return [];
}
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index ab58050eb..cdb39998b 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -1,4 +1,5 @@
import "dotenv/config";
+import { config } from "../../config";
import "../sentry";
import { setSentryServiceTag } from "../sentry";
import * as Sentry from "@sentry/node";
@@ -42,19 +43,14 @@ import { getACUCTeam } from "../../controllers/auth";
import { supabase_service } from "../supabase";
import { processEngpickerJob } from "../../lib/engpicker";
-const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;
-const workerStalledCheckInterval =
- Number(process.env.WORKER_STALLED_CHECK_INTERVAL) || 30000;
-const jobLockExtendInterval =
- Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 15000;
-const jobLockExtensionTime =
- Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;
+const workerLockDuration = config.WORKER_LOCK_DURATION;
+const workerStalledCheckInterval = config.WORKER_STALLED_CHECK_INTERVAL;
+const jobLockExtendInterval = config.JOB_LOCK_EXTEND_INTERVAL;
+const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;
-const cantAcceptConnectionInterval =
- Number(process.env.CANT_ACCEPT_CONNECTION_INTERVAL) || 2000;
-const connectionMonitorInterval =
- Number(process.env.CONNECTION_MONITOR_INTERVAL) || 10;
-const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;
+const cantAcceptConnectionInterval = config.CANT_ACCEPT_CONNECTION_INTERVAL;
+const connectionMonitorInterval = config.CONNECTION_MONITOR_INTERVAL;
+const gotJobInterval = config.CONNECTION_MONITOR_INTERVAL;
const runningJobs: Set<string> = new Set();
@@ -150,7 +146,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {
const MIN_URLS_PER_DOMAIN = 10;
const MAX_URLS_PER_DOMAIN = 250;
- const teamId = process.env.PRECRAWL_TEAM_ID;
+ const teamId = config.PRECRAWL_TEAM_ID;
try {
await withSpan("precrawl.job", async span => {
@@ -470,8 +466,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {
internalOptions: {
disableSmartWaitCache: true, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter
teamId,
- saveScrapeResultToGCS:
- !!process.env.GCS_FIRE_ENGINE_BUCKET_NAME,
+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
zeroDataRetention: false,
isPreCrawl: true, // NOTE: must be added to internal options for indexing, if not it will be treated as a normal scrape in the index
},
@@ -692,7 +687,7 @@ const DOMAIN_FREQUENCY_INTERVAL = 10000;
processBillingJobInternal,
);
- const precrawlWorkerPromise = process.env.PRECRAWL_TEAM_ID
+ const precrawlWorkerPromise = config.PRECRAWL_TEAM_ID
? workerFun(getPrecrawlQueue(), processPrecrawlJob)
: (async () => {
logger.warn("PRECRAWL_TEAM_ID not set, skipping precrawl worker");
diff --git a/apps/api/src/services/ledger/supabase-ledger.ts b/apps/api/src/services/ledger/supabase-ledger.ts
index 434214910..7767aded2 100644
--- a/apps/api/src/services/ledger/supabase-ledger.ts
+++ b/apps/api/src/services/ledger/supabase-ledger.ts
@@ -1,6 +1,7 @@
import { createClient, SupabaseClient } from "@supabase/supabase-js";
import { logger } from "../../lib/logger";
import { configDotenv } from "dotenv";
+import { config } from "../../config";
configDotenv();
// SupabaseLedgerService class initializes the Supabase client for the ledger schema
@@ -9,10 +10,10 @@ class SupabaseLedgerService {
private rrClient: SupabaseClient<any, "ledger", any> | null = null;
constructor() {
- const supabaseUrl = process.env.SUPABASE_URL;
- const supabaseReplicaUrl = process.env.SUPABASE_REPLICA_URL;
- const supabaseServiceToken = process.env.SUPABASE_SERVICE_TOKEN;
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
+ const supabaseUrl = config.SUPABASE_URL;
+ const supabaseReplicaUrl = config.SUPABASE_REPLICA_URL;
+ const supabaseServiceToken = config.SUPABASE_SERVICE_TOKEN;
+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;
// Only initialize the Supabase client if both URL and Service Token are provided.
if (!useDbAuthentication) {
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 0515e37b7..4361498a4 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -1,4 +1,5 @@
import { supabase_service } from "../supabase";
+import { config } from "../../config";
import "dotenv/config";
import { logger as _logger } from "../../lib/logger";
import { configDotenv } from "dotenv";
@@ -24,7 +25,7 @@ async function robustInsert(
force: boolean,
logger: Logger,
) {
- if (process.env.USE_DB_AUTHENTICATION !== "true") {
+ if (config.USE_DB_AUTHENTICATION !== true) {
logger.info(
"Skipping database insertion due to USE_DB_AUTHENTICATION being off",
{ table },
@@ -193,7 +194,7 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
if (
scrape.doc &&
- process.env.GCS_BUCKET_NAME &&
+ config.GCS_BUCKET_NAME &&
!(scrape.skipNuq && scrape.zeroDataRetention)
) {
await saveScrapeToGCS(scrape);
@@ -202,7 +203,7 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
if (
scrape.is_successful &&
!scrape.zeroDataRetention &&
- process.env.USE_DB_AUTHENTICATION === "true"
+ config.USE_DB_AUTHENTICATION
) {
const hasMarkdown = hasFormatOfType(scrape.options.formats, "markdown");
const hasChangeTracking = hasFormatOfType(
diff --git a/apps/api/src/services/notification/email_notification.ts b/apps/api/src/services/notification/email_notification.ts
index 303efbb42..4f7f871d5 100644
--- a/apps/api/src/services/notification/email_notification.ts
+++ b/apps/api/src/services/notification/email_notification.ts
@@ -1,4 +1,5 @@
import { supabase_service } from "../supabase";
+import { config } from "../../config";
import { withAuth } from "../../lib/withAuth";
import { Resend } from "resend";
import { NotificationType } from "../../types";
@@ -104,7 +105,7 @@ async function sendEmailNotification(
notificationType: NotificationType,
context: NotificationContext = {},
) {
- const resend = new Resend(process.env.RESEND_API_KEY);
+ const resend = new Resend(config.RESEND_API_KEY);
try {
// Get user's email preferences
@@ -297,11 +298,11 @@ async function sendNotificationInternal(
},
]);
- if (process.env.SLACK_ADMIN_WEBHOOK_URL && emails.length > 0) {
+ if (config.SLACK_ADMIN_WEBHOOK_URL && emails.length > 0) {
sendSlackWebhook(
`${getNotificationString(notificationType)}: Team ${team_id}, with email ${emails[0].email}. Number of credits used: ${chunk.adjusted_credits_used} | Number of credits in the plan: ${chunk.price_credits}`,
false,
- process.env.SLACK_ADMIN_WEBHOOK_URL,
+ config.SLACK_ADMIN_WEBHOOK_URL,
).catch(error => {
logger.debug(`Error sending slack notification: ${error}`);
});
@@ -428,14 +429,14 @@ export async function sendNotificationWithCustomDays(
]);
if (
- process.env.SLACK_ADMIN_WEBHOOK_URL &&
+ config.SLACK_ADMIN_WEBHOOK_URL &&
emails.length > 0 &&
notificationType !== NotificationType.CONCURRENCY_LIMIT_REACHED
) {
sendSlackWebhook(
`${getNotificationString(notificationType)}: Team ${team_id}, with email ${emails[0].email}.`,
false,
- process.env.SLACK_ADMIN_WEBHOOK_URL,
+ config.SLACK_ADMIN_WEBHOOK_URL,
).catch(error => {
logger.debug(`Error sending slack notification: ${error}`);
});
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 6bd80afe6..465ec6761 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -1,4 +1,5 @@
import { Queue } from "bullmq";
+import { config } from "../config";
import { logger } from "../lib/logger";
import IORedis from "ioredis";
import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";
@@ -14,7 +15,7 @@ let redisConnection: IORedis;
export function getRedisConnection(): IORedis {
if (!redisConnection) {
- redisConnection = new IORedis(process.env.REDIS_URL!, {
+ redisConnection = new IORedis(config.REDIS_URL!, {
maxRetriesPerRequest: null,
});
redisConnection.on("connect", () => logger.info("Redis connected"));
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 53e9cd227..3bda3f5b3 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -1,4 +1,5 @@
import "dotenv/config";
+import { config } from "../config";
import "./sentry";
import { setSentryServiceTag } from "./sentry";
import * as Sentry from "@sentry/node";
@@ -29,16 +30,12 @@ configDotenv();
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
-const jobLockExtendInterval =
- Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;
-const jobLockExtensionTime =
- Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;
+const jobLockExtendInterval = config.JOB_LOCK_EXTEND_INTERVAL;
+const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;
-const cantAcceptConnectionInterval =
- Number(process.env.CANT_ACCEPT_CONNECTION_INTERVAL) || 2000;
-const connectionMonitorInterval =
- Number(process.env.CONNECTION_MONITOR_INTERVAL) || 10;
-const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;
+const cantAcceptConnectionInterval = config.CANT_ACCEPT_CONNECTION_INTERVAL;
+const connectionMonitorInterval = config.CONNECTION_MONITOR_INTERVAL;
+const gotJobInterval = config.CONNECTION_MONITOR_INTERVAL;
const runningJobs: Set<string> = new Set();
@@ -328,7 +325,7 @@ const crawlFinishWorker = async () => {
if (job === null) {
__logger.info("No jobs to process", { module: "nuq/metrics" });
await new Promise(resolve => setTimeout(resolve, noJobTimeout));
- if (!process.env.NUQ_RABBITMQ_URL) {
+ if (!config.NUQ_RABBITMQ_URL) {
noJobTimeout = Math.min(noJobTimeout * 2, 10000);
}
continue;
@@ -408,11 +405,11 @@ let currentLiveness: boolean = true;
app.get("/liveness", (req, res) => {
_logger.info("Liveness endpoint hit");
- if (process.env.USE_DB_AUTHENTICATION === "true") {
+ if (config.USE_DB_AUTHENTICATION) {
// networking check for Kubernetes environments
- const host = process.env.FIRECRAWL_APP_HOST || "firecrawl-app-service";
- const port = process.env.FIRECRAWL_APP_PORT || "3002";
- const scheme = process.env.FIRECRAWL_APP_SCHEME || "http";
+ const host = config.FIRECRAWL_APP_HOST;
+ const port = config.FIRECRAWL_APP_PORT;
+ const scheme = config.FIRECRAWL_APP_SCHEME;
robustFetch({
url: `${scheme}://${host}:${port}`,
@@ -438,7 +435,7 @@ app.get("/liveness", (req, res) => {
}
});
-const workerPort = process.env.WORKER_PORT || process.env.PORT || 3005;
+const workerPort = config.WORKER_PORT || config.PORT;
app.listen(workerPort, () => {
_logger.info(`Liveness endpoint is running on port ${workerPort}`);
});
diff --git a/apps/api/src/services/rate-limiter.test.ts b/apps/api/src/services/rate-limiter.test.ts
index 5e8e067f2..709b32904 100644
--- a/apps/api/src/services/rate-limiter.test.ts
+++ b/apps/api/src/services/rate-limiter.test.ts
@@ -1,3 +1,4 @@
+import { config } from "../config";
// import {
// getRateLimiter,
// serverRateLimiter,
@@ -10,7 +11,7 @@
// beforeAll(async () => {
// try {
// await redisRateLimitClient.connect();
-// // if (process.env.REDIS_RATE_LIMIT_URL === "redis://localhost:6379") {
+// // if (config.REDIS_RATE_LIMIT_URL === "redis://localhost:6379") {
// // console.log("Erasing all keys");
// // // erase all the keys that start with "test-prefix"
// // const keys = await redisRateLimitClient.keys("test-prefix:*");
@@ -23,7 +24,7 @@
// afterAll(async () => {
// try {
-// // if (process.env.REDIS_RATE_LIMIT_URL === "redis://localhost:6379") {
+// // if (config.REDIS_RATE_LIMIT_URL === "redis://localhost:6379") {
// await redisRateLimitClient.disconnect();
// // }
// } catch (error) {}
diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts
index 2b6bbda05..deea0e867 100644
--- a/apps/api/src/services/rate-limiter.ts
+++ b/apps/api/src/services/rate-limiter.ts
@@ -1,11 +1,10 @@
import { RateLimiterRedis } from "rate-limiter-flexible";
+import { config } from "../config";
import { RateLimiterMode } from "../types";
import Redis from "ioredis";
import type { AuthCreditUsageChunk } from "../controllers/v1/types";
-export const redisRateLimitClient = new Redis(
- process.env.REDIS_RATE_LIMIT_URL!,
-);
+export const redisRateLimitClient = new Redis(config.REDIS_RATE_LIMIT_URL!);
const createRateLimiter = (keyPrefix, points) =>
new RateLimiterRedis({
diff --git a/apps/api/src/services/redis.ts b/apps/api/src/services/redis.ts
index af5f9f7ca..7049020c0 100644
--- a/apps/api/src/services/redis.ts
+++ b/apps/api/src/services/redis.ts
@@ -1,4 +1,5 @@
import IORedis from "ioredis";
+import { config } from "../config";
import { redisRateLimitClient } from "./rate-limiter";
import { logger } from "../lib/logger";
@@ -71,6 +72,5 @@ const deleteKey = async (key: string) => {
export { setValue, getValue, deleteKey };
-const redisEvictURL =
- process.env.REDIS_EVICT_URL ?? process.env.REDIS_RATE_LIMIT_URL;
+const redisEvictURL = config.REDIS_EVICT_URL ?? config.REDIS_RATE_LIMIT_URL;
export const redisEvictConnection = new IORedis(redisEvictURL!);
diff --git a/apps/api/src/services/redlock.ts b/apps/api/src/services/redlock.ts
index 923cfc3d0..25a9767ce 100644
--- a/apps/api/src/services/redlock.ts
+++ b/apps/api/src/services/redlock.ts
@@ -1,10 +1,11 @@
import Redlock from "redlock";
+import { config } from "../config";
import Client from "ioredis";
export const redlock = new Redlock(
// You should have one client for each independent redis node
// or cluster.
- [new Client(process.env.REDIS_RATE_LIMIT_URL!)],
+ [new Client(config.REDIS_RATE_LIMIT_URL!)],
{
// The expected clock drift; for more details see:
// http://redis.io/topics/distlock
diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.ts
index 1c4a7fecd..d7e51838c 100644
--- a/apps/api/src/services/sentry.ts
+++ b/apps/api/src/services/sentry.ts
@@ -1,18 +1,12 @@
import * as Sentry from "@sentry/node";
import { logger } from "../lib/logger";
+import { config } from "../config";
-if (process.env.SENTRY_DSN) {
+if (config.SENTRY_DSN) {
logger.info("Setting up Sentry...");
- const TRACE_SAMPLE_RATE = parseFloat(
- process.env.SENTRY_TRACE_SAMPLE_RATE || "0",
- );
- const ERROR_SAMPLE_RATE = parseFloat(
- process.env.SENTRY_ERROR_SAMPLE_RATE || "0.05",
- );
-
Sentry.init({
- dsn: process.env.SENTRY_DSN,
+ dsn: config.SENTRY_DSN,
integrations: integrations => [
...integrations,
Sentry.vercelAIIntegration({
@@ -22,11 +16,13 @@ if (process.env.SENTRY_DSN) {
],
tracesSampler: samplingContext => {
// trace all AI spans, sample 1% of all others
- return samplingContext.name?.startsWith("ai.") ? 1.0 : TRACE_SAMPLE_RATE;
+ return samplingContext.name?.startsWith("ai.")
+ ? 1.0
+ : config.SENTRY_TRACE_SAMPLE_RATE;
},
- sampleRate: ERROR_SAMPLE_RATE,
- serverName: process.env.NUQ_POD_NAME,
- environment: process.env.SENTRY_ENVIRONMENT ?? "production",
+ sampleRate: config.SENTRY_ERROR_SAMPLE_RATE,
+ serverName: config.NUQ_POD_NAME,
+ environment: config.SENTRY_ENVIRONMENT,
beforeSend(event, hint) {
const error = hint?.originalException;
@@ -70,7 +66,7 @@ if (process.env.SENTRY_DSN) {
* This helps distinguish between API server and worker errors in Sentry
*/
export function setSentryServiceTag(serviceType: string) {
- if (process.env.SENTRY_DSN) {
+ if (config.SENTRY_DSN) {
Sentry.setTag("service_type", serviceType);
}
}
diff --git a/apps/api/src/services/supabase.ts b/apps/api/src/services/supabase.ts
index 62a08baf1..71dc4991d 100644
--- a/apps/api/src/services/supabase.ts
+++ b/apps/api/src/services/supabase.ts
@@ -1,4 +1,5 @@
import { createClient, SupabaseClient } from "@supabase/supabase-js";
+import { config } from "../config";
import { logger } from "../lib/logger";
import { configDotenv } from "dotenv";
configDotenv();
@@ -11,16 +12,15 @@ class SupabaseService {
private acucClient: SupabaseClient | null = null;
constructor() {
- const supabaseUrl = process.env.SUPABASE_URL;
- const supabaseReplicaUrl = process.env.SUPABASE_REPLICA_URL;
- const supabaseServiceToken = process.env.SUPABASE_SERVICE_TOKEN;
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
+ const supabaseUrl = config.SUPABASE_URL;
+ const supabaseReplicaUrl = config.SUPABASE_REPLICA_URL;
+ const supabaseServiceToken = config.SUPABASE_SERVICE_TOKEN;
+ const useDbAuthentication = config.USE_DB_AUTHENTICATION;
// Only initialize the Supabase client if both URL and Service Token are provided.
if (!useDbAuthentication) {
- if (!!process.env.SUPABASE_ACUC_URL) {
- const supabaseAcucUrl = process.env.SUPABASE_ACUC_URL;
- const supabaseAcucServiceToken =
- process.env.SUPABASE_ACUC_SERVICE_TOKEN;
+ if (!!config.SUPABASE_ACUC_URL) {
+ const supabaseAcucUrl = config.SUPABASE_ACUC_URL;
+ const supabaseAcucServiceToken = config.SUPABASE_ACUC_SERVICE_TOKEN;
if (!supabaseAcucUrl || !supabaseAcucServiceToken) {
logger.error(
"Supabase ACUC environment variables aren't configured correctly. Supabase ACUC client will not be initialized. Fix ENV configuration or disable ACUC with USE_DB_AUTHENTICATION env variable",
diff --git a/apps/api/src/services/system-monitor.ts b/apps/api/src/services/system-monitor.ts
index 23f6d396b..afe807cae 100644
--- a/apps/api/src/services/system-monitor.ts
+++ b/apps/api/src/services/system-monitor.ts
@@ -1,16 +1,15 @@
import si from "systeminformation";
+import { config } from "../config";
import { Mutex } from "async-mutex";
import os from "os";
import fs from "fs";
import { logger } from "../lib/logger";
-const IS_KUBERNETES = process.env.IS_KUBERNETES === "true";
+const IS_KUBERNETES = config.IS_KUBERNETES;
-const MAX_CPU = process.env.MAX_CPU ? parseFloat(process.env.MAX_CPU) : 0.8;
-const MAX_RAM = process.env.MAX_RAM ? parseFloat(process.env.MAX_RAM) : 0.8;
-const CACHE_DURATION = process.env.SYS_INFO_MAX_CACHE_DURATION
- ? parseFloat(process.env.SYS_INFO_MAX_CACHE_DURATION)
- : 150;
+const MAX_CPU = config.MAX_CPU;
+const MAX_RAM = config.MAX_RAM;
+const CACHE_DURATION = config.SYS_INFO_MAX_CACHE_DURATION;
class SystemMonitor {
private static instance: SystemMonitor;
diff --git a/apps/api/src/services/webhook/config.ts b/apps/api/src/services/webhook/config.ts
index 36f1dc784..5b35a4314 100644
--- a/apps/api/src/services/webhook/config.ts
+++ b/apps/api/src/services/webhook/config.ts
@@ -2,6 +2,7 @@ import { logger as _logger } from "../../lib/logger";
import { supabase_rr_service } from "../supabase";
import { WebhookConfig } from "./types";
+import { config } from "../../config";
export async function getWebhookConfig(
teamId: string,
jobId: string,
@@ -15,7 +16,7 @@ export async function getWebhookConfig(
return { config: webhook, secret: await getHmacSecret(teamId) };
}
- const selfHostedUrl = process.env.SELF_HOSTED_WEBHOOK_URL?.replace(
+ const selfHostedUrl = config.SELF_HOSTED_WEBHOOK_URL?.replace(
"{{JOB_ID}}",
jobId,
);
@@ -27,11 +28,11 @@ export async function getWebhookConfig(
metadata: {},
events: ["completed", "failed", "page", "started"],
},
- secret: process.env.SELF_HOSTED_WEBHOOK_HMAC_SECRET,
+ secret: config.SELF_HOSTED_WEBHOOK_HMAC_SECRET,
};
}
- if (process.env.USE_DB_AUTHENTICATION === "true") {
+ if (config.USE_DB_AUTHENTICATION) {
const dbConfig = await fetchWebhookFromDb(teamId);
if (dbConfig) {
return { config: dbConfig, secret: await getHmacSecret(teamId) };
@@ -59,8 +60,8 @@ async function fetchWebhookFromDb(
}
async function getHmacSecret(teamId: string): Promise<string | undefined> {
- if (process.env.USE_DB_AUTHENTICATION !== "true") {
- return process.env.SELF_HOSTED_WEBHOOK_HMAC_SECRET;
+ if (config.USE_DB_AUTHENTICATION !== true) {
+ return config.SELF_HOSTED_WEBHOOK_HMAC_SECRET;
}
try {
diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.ts
index 4fe61e6f6..f91ac0d54 100644
--- a/apps/api/src/services/webhook/delivery.ts
+++ b/apps/api/src/services/webhook/delivery.ts
@@ -1,4 +1,5 @@
import undici from "undici";
+import { config } from "../../config";
import { createHmac } from "crypto";
import { logger as _logger, logger } from "../../lib/logger";
import {
@@ -62,7 +63,7 @@ export class WebhookSender {
}
private shouldSendEvent(event: WebhookEvent): boolean {
- if (process.env.DISABLE_WEBHOOK_DELIVERY === "true") {
+ if (config.DISABLE_WEBHOOK_DELIVERY) {
return false;
}
@@ -76,10 +77,7 @@ export class WebhookSender {
private async deliver(payload: any, scrapeId?: string): Promise<void> {
const webhookHost = new URL(this.config.url).hostname;
- if (
- isIPPrivate(webhookHost) &&
- process.env.ALLOW_LOCAL_WEBHOOKS !== "true"
- ) {
+ if (isIPPrivate(webhookHost) && config.ALLOW_LOCAL_WEBHOOKS !== true) {
this.logger.warn("Aborting webhook call to private IP address", {
webhookUrl: this.config.url,
});
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index 06dd1960a..35516137f 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -1,4 +1,5 @@
import { logger as _logger } from "../../lib/logger";
+import { config } from "../../config";
import {
finishCrawl,
getCrawlJobs,
@@ -128,7 +129,7 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
let credits_billed = null;
- if (process.env.USE_DB_AUTHENTICATION === "true") {
+ if (config.USE_DB_AUTHENTICATION) {
const creditsRpc = await supabase_service.rpc(
"credits_billed_by_crawl_id_2",
{
diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.ts
index c2f329a49..909bd10f4 100644
--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts
+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts
@@ -1,4 +1,5 @@
import "dotenv/config";
+import { config } from "../../config";
import "../sentry";
import { setSentryServiceTag } from "../sentry";
import {
@@ -27,12 +28,9 @@ import { logger } from "../../lib/logger";
}
});
- const server = app.listen(
- Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011),
- () => {
- logger.info("NuQ prefetch worker metrics server started");
- },
- );
+ const server = app.listen(config.NUQ_PREFETCH_WORKER_PORT, () => {
+ logger.info("NuQ prefetch worker metrics server started");
+ });
async function shutdown() {
server.close();
diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.ts
index 883f0ee50..cb4aaf0ec 100644
--- a/apps/api/src/services/worker/nuq-worker.ts
+++ b/apps/api/src/services/worker/nuq-worker.ts
@@ -1,4 +1,5 @@
import "dotenv/config";
+import { config } from "../../config";
import "../sentry";
import { setSentryServiceTag } from "../sentry";
import { logger as _logger } from "../../lib/logger";
@@ -37,12 +38,9 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f
}
});
- const server = app.listen(
- Number(process.env.NUQ_WORKER_PORT ?? process.env.PORT ?? 3000),
- () => {
- _logger.info("NuQ worker metrics server started");
- },
- );
+ const server = app.listen(config.NUQ_WORKER_PORT, () => {
+ _logger.info("NuQ worker metrics server started");
+ });
function shutdown() {
isShuttingDown = true;
@@ -59,7 +57,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f
if (job === null) {
_logger.info("No jobs to process", { module: "nuq/metrics" });
await new Promise(resolve => setTimeout(resolve, noJobTimeout));
- if (!process.env.NUQ_RABBITMQ_URL) {
+ if (!config.NUQ_RABBITMQ_URL) {
noJobTimeout = Math.min(noJobTimeout * 2, 10000);
}
continue;
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index 6f737ed5f..12e366c11 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -5,11 +5,12 @@ import { type ScrapeJobData } from "../../types";
import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
import amqp from "amqplib";
import { v5 as uuidv5, validate as isUUID } from "uuid";
+import { config } from "../../config";
// === Basics
const nuqPool = new Pool({
- connectionString: process.env.NUQ_DATABASE_URL, // may be a pgbouncer transaction pooler URL
+ connectionString: config.NUQ_DATABASE_URL, // may be a pgbouncer transaction pooler URL
application_name: "nuq",
});
@@ -63,7 +64,7 @@ function normalizeOwnerId(ownerId: string | undefined | null): string | null {
class NuQ<JobData = any, JobReturnValue = any> {
private listenChannelId: string =
- (process.env.NUQ_POD_NAME ?? "main") + "-" + crypto.randomUUID();
+ config.NUQ_POD_NAME + "-" + crypto.randomUUID();
constructor(
public readonly queueName: string,
@@ -93,11 +94,11 @@ class NuQ<JobData = any, JobReturnValue = any> {
private async startListener() {
if (this.listener || this.shuttingDown || this.listenerStarting) return;
- if (process.env.NUQ_RABBITMQ_URL) {
+ if (config.NUQ_RABBITMQ_URL) {
this.listenerStarting = true;
try {
- const connection = await amqp.connect(process.env.NUQ_RABBITMQ_URL);
+ const connection = await amqp.connect(config.NUQ_RABBITMQ_URL);
const channel = await connection.createChannel();
await channel.prefetch(1);
const queue = await channel.assertQueue(
@@ -184,7 +185,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
type: "postgres",
client: new Client({
connectionString:
- process.env.NUQ_DATABASE_URL_LISTEN ?? process.env.NUQ_DATABASE_URL, // will always be a direct connection
+ config.NUQ_DATABASE_URL_LISTEN ?? config.NUQ_DATABASE_URL, // will always be a direct connection
application_name: "nuq_listener",
}),
};
@@ -267,8 +268,8 @@ class NuQ<JobData = any, JobReturnValue = any> {
private async startSender() {
if (this.sender || this.shuttingDown) return;
- if (process.env.NUQ_RABBITMQ_URL) {
- const connection = await amqp.connect(process.env.NUQ_RABBITMQ_URL);
+ if (config.NUQ_RABBITMQ_URL) {
+ const connection = await amqp.connect(config.NUQ_RABBITMQ_URL);
const channel = await connection.createChannel();
await channel.assertQueue(this.queueName + ".prefetch", {
durable: true,
@@ -1038,7 +1039,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
private readonly nuqWaitMode =
- process.env.NUQ_WAIT_MODE === "listen" || process.env.NUQ_RABBITMQ_URL
+ config.NUQ_WAIT_MODE === "listen" || config.NUQ_RABBITMQ_URL
? ("listen" as const)
: ("poll" as const);
@@ -1194,7 +1195,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
public async getJobToProcess(): Promise<NuQJob<any, any> | null> {
const start = Date.now();
try {
- if (process.env.NUQ_RABBITMQ_URL) {
+ if (config.NUQ_RABBITMQ_URL) {
await this.startSender();
if (this.sender) {
@@ -1281,11 +1282,11 @@ class NuQ<JobData = any, JobReturnValue = any> {
if (success) {
const job = result.rows[0];
- if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {
+ if (this.nuqWaitMode === "listen" && !config.NUQ_RABBITMQ_URL) {
await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [
job.id + "|completed",
]);
- } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {
+ } else if (config.NUQ_RABBITMQ_URL && job.listen_channel_id) {
await this.sendJobEnd(
job.id,
"completed",
@@ -1339,11 +1340,11 @@ class NuQ<JobData = any, JobReturnValue = any> {
if (success) {
const job = result.rows[0];
- if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {
+ if (this.nuqWaitMode === "listen" && !config.NUQ_RABBITMQ_URL) {
await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [
job.id + "|failed",
]);
- } else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {
+ } else if (config.NUQ_RABBITMQ_URL && job.listen_channel_id) {
await this.sendJobEnd(
job.id,
"failed",
diff --git a/apps/api/src/services/worker/redis.ts b/apps/api/src/services/worker/redis.ts
index bca1aa2a8..c98859152 100644
--- a/apps/api/src/services/worker/redis.ts
+++ b/apps/api/src/services/worker/redis.ts
@@ -1,4 +1,5 @@
import IORedis, { Redis } from "ioredis";
+import { config } from "../../config";
import { logger as _logger } from "../../lib/logger";
const logger = _logger.child({ module: "nuq/redis" });
@@ -57,7 +58,7 @@ type ScriptHashes = {
const scripts: ScriptHashes = {} as ScriptHashes;
-const redis = new IORedis(process.env.REDIS_URL!, {
+const redis = new IORedis(config.REDIS_URL!, {
lazyConnect: true,
maxRetriesPerRequest: null,
enableReadyCheck: false,
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 6c4d2deb7..0d665da87 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -1,4 +1,5 @@
import { configDotenv } from "dotenv";
+import { config } from "../../config";
import * as Sentry from "@sentry/node";
import http from "http";
import https from "https";
@@ -76,10 +77,8 @@ import { FeatureFlag } from "../../scraper/scrapeURL/engines";
configDotenv();
-const jobLockExtendInterval =
- Number(process.env.JOB_LOCK_EXTEND_INTERVAL) || 10000;
-const jobLockExtensionTime =
- Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;
+const jobLockExtendInterval = config.JOB_LOCK_EXTEND_INTERVAL;
+const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;
if (require.main === module) {
cacheableLookup.install(http.globalAgent);
@@ -109,8 +108,8 @@ async function billScrapeJob(
);
if (
- job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&
- process.env.USE_DB_AUTHENTICATION === "true"
+ job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&
+ config.USE_DB_AUTHENTICATION
) {
try {
const billingJobId = uuidv7();
@@ -1165,7 +1164,7 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
}
try {
- if (process.env.GCS_BUCKET_NAME && !job.data.skipNuq) {
+ if (config.GCS_BUCKET_NAME && !job.data.skipNuq) {
logger.debug("Job succeeded -- putting null in Redis");
return null;
} else {