Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Optimize Logs File Size (#2456)

Abimael Martell784d1063d453
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index bce12dbc7..63a209d2e 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -44,6 +44,7 @@ jobs:
NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
USE_GO_MARKDOWN_PARSER: true
ALLOW_LOCAL_WEBHOOKS: true
+ FIRECRAWL_LOG_TO_FILE: true
steps:
- uses: actions/checkout@v5
@@ -201,7 +202,8 @@ jobs:
if: always()
run: |
mkdir -p logs
- cp ./apps/api/firecrawl.log logs/firecrawl.log
+ cp ./apps/api/firecrawl-app.log logs/firecrawl.log || true
+ cp ./apps/api/firecrawl-worker.log logs/firecrawl-worker.log || true
- name: Copy SearXNG logs
if: always() && matrix.search == 'searxng'
diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.ts
index 9bbb17c47..db853250c 100644
--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts
@@ -1,5 +1,4 @@
import { getJobFromGCS } from "../../../lib/gcs-jobs";
-import { supabase_service } from "../../../services/supabase";
import {
scrape,
crawl,
@@ -8,51 +7,13 @@ import {
zdrcleaner,
idmux,
} from "./lib";
-import { readFile, stat } from "node:fs/promises";
import { describeIf, TEST_PRODUCTION } from "../lib";
-
-const logIgnoreList = [
- "Billing queue created",
- "No billing operations to process in batch",
- "billing batch queue",
- "billing batch processing lock",
- "Batch billing team",
- "Successfully billed team",
- "Billing batch processing",
- "Processing batch of",
- "Billing team",
- "No jobs to process",
- "nuqHealthCheck metrics",
- "nuqGetJobToProcess metrics",
- "Domain frequency processor",
- "billing operation to batch queue",
- "billing operation to queue",
- "billing operation for team",
- "Added billing operation to queue",
- "Index RF inserter found",
- "Redis connected",
- "Prefetched jobs",
- "nuqPrefetchJobs metrics",
- "request completed",
- "nuqAddJobs metrics",
- "nuqGetJobs metrics",
-];
-
-async function getLogs() {
- let logs: string;
- try {
- await stat("firecrawl.log");
- } catch (e) {
- console.warn("No firecrawl.log file found");
- return [];
- }
- logs = await readFile("firecrawl.log", "utf8");
- return logs
- .split("\n")
- .filter(
- x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y)),
- );
-}
+import {
+ getLogs,
+ expectJobRecordIsCleanedUp,
+ expectJobsArrayIsCleanedUp,
+ expectJobsAreFullyCleanedAfterZDRCleaner,
+} from "../zdr-helpers";
describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
describe.each(["Team-scoped", "Request-scoped"] as const)("%s", scope => {
@@ -82,22 +43,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!);
expect(gcsJob).toBeNull();
- const { data, error } = await supabase_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("job_id", scrape1.metadata.scrapeId!)
- .limit(1);
-
- expect(error).toBeFalsy();
- expect(data).toHaveLength(1);
-
- if (data && data.length === 1) {
- const record = data[0];
- expect(record.url).not.toContain("://"); // no url stored
- expect(record.docs).toBeNull();
- expect(record.page_options).toBeNull();
- expect(record.crawler_options).toBeNull();
- }
+ await expectJobRecordIsCleanedUp(scrape1.metadata.scrapeId!);
if (scope === "Request-scoped") {
const status = await scrapeStatusRaw(
@@ -144,55 +90,18 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
expect(postLogs).toHaveLength(0);
- const { data, error } = await supabase_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("job_id", crawl1.id)
- .limit(1);
-
- expect(error).toBeFalsy();
- expect(data).toHaveLength(1);
+ await expectJobRecordIsCleanedUp(crawl1.id);
- if (data && data.length === 1) {
- const record = data[0];
- expect(record.url).not.toContain("://"); // no url stored
- expect(record.docs).toBeNull();
- expect(record.page_options).toBeNull();
- expect(record.crawler_options).toBeNull();
- }
-
- const { data: jobs, error: jobsError } = await supabase_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("crawl_id", crawl1.id);
-
- expect(jobsError).toBeFalsy();
- expect((jobs ?? []).length).toBeGreaterThanOrEqual(1);
-
- for (const job of jobs ?? []) {
- expect(job.url).not.toContain("://"); // no url stored
- expect(job.docs).toBeNull();
- expect(job.page_options).toBeNull();
- expect(job.crawler_options).toBeNull();
- expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry
-
- if (job.success) {
- const gcsJob = await getJobFromGCS(job.job_id);
- expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry
- }
- }
+ const jobs = await expectJobsArrayIsCleanedUp(crawl1.id);
await zdrcleaner(identity.teamId!);
- for (const job of jobs ?? []) {
- const gcsJob = await getJobFromGCS(job.job_id);
- expect(gcsJob).toBeNull();
-
- if (scope === "Request-scoped") {
- const status = await scrapeStatusRaw(job.job_id, identity);
- expect(status.statusCode).toBe(404);
- }
- }
+ await expectJobsAreFullyCleanedAfterZDRCleaner(
+ jobs,
+ scope,
+ identity,
+ scrapeStatusRaw,
+ );
}, 600000);
it("should clean up a batch scrape", async () => {
@@ -229,55 +138,18 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
expect(postLogs).toHaveLength(0);
- const { data, error } = await supabase_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("job_id", crawl1.id)
- .limit(1);
-
- expect(error).toBeFalsy();
- expect(data).toHaveLength(1);
-
- if (data && data.length === 1) {
- const record = data[0];
- expect(record.url).not.toContain("://"); // no url stored
- expect(record.docs).toBeNull();
- expect(record.page_options).toBeNull();
- expect(record.crawler_options).toBeNull();
- }
-
- const { data: jobs, error: jobsError } = await supabase_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("crawl_id", crawl1.id);
+ await expectJobRecordIsCleanedUp(crawl1.id);
- expect(jobsError).toBeFalsy();
- expect((jobs ?? []).length).toBe(2);
-
- for (const job of jobs ?? []) {
- expect(job.url).not.toContain("://"); // no url stored
- expect(job.docs).toBeNull();
- expect(job.page_options).toBeNull();
- expect(job.crawler_options).toBeNull();
- expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry
-
- if (job.success) {
- const gcsJob = await getJobFromGCS(job.job_id);
- expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry
- }
- }
+ const jobs = await expectJobsArrayIsCleanedUp(crawl1.id, 2);
await zdrcleaner(identity.teamId!);
- for (const job of jobs ?? []) {
- const gcsJob = await getJobFromGCS(job.job_id);
- expect(gcsJob).toBeNull();
-
- if (scope === "Request-scoped") {
- const status = await scrapeStatusRaw(job.job_id, identity);
- expect(status.statusCode).toBe(404);
- }
- }
+ await expectJobsAreFullyCleanedAfterZDRCleaner(
+ jobs,
+ scope,
+ identity,
+ scrapeStatusRaw,
+ );
}, 600000);
});
});
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.ts
index 3687a66ad..dcccbe48a 100644
--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts
@@ -1,4 +1,3 @@
-import { supabase_service } from "../../../services/supabase";
import { getJobFromGCS } from "../../../lib/gcs-jobs";
import {
scrape,
@@ -8,51 +7,13 @@ import {
zdrcleaner,
idmux,
} from "./lib";
-import { readFile, stat } from "node:fs/promises";
import { describeIf, TEST_PRODUCTION } from "../lib";
-
-const logIgnoreList = [
- "Billing queue created",
- "No billing operations to process in batch",
- "billing batch queue",
- "billing batch processing lock",
- "Batch billing team",
- "Successfully billed team",
- "Billing batch processing",
- "Processing batch of",
- "Billing team",
- "No jobs to process",
- "nuqHealthCheck metrics",
- "nuqGetJobToProcess metrics",
- "Domain frequency processor",
- "billing operation to batch queue",
- "billing operation to queue",
- "billing operation for team",
- "Added billing operation to queue",
- "Index RF inserter found",
- "Redis connected",
- "Prefetched jobs",
- "nuqPrefetchJobs metrics",
- "request completed",
- "nuqAddJobs metrics",
- "nuqGetJobs metrics",
-];
-
-async function getLogs() {
- let logs: string;
- try {
- await stat("firecrawl.log");
- } catch (e) {
- console.warn("No firecrawl.log file found");
- return [];
- }
- logs = await readFile("firecrawl.log", "utf8");
- return logs
- .split("\n")
- .filter(
- x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y)),
- );
-}
+import {
+ getLogs,
+ expectJobRecordIsCleanedUp,
+ expectJobsArrayIsCleanedUp,
+ expectJobsAreFullyCleanedAfterZDRCleaner,
+} from "../zdr-helpers";
describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
describe.each(["Team-scoped", "Request-scoped"] as const)("%s", scope => {
@@ -82,22 +43,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!);
expect(gcsJob).toBeNull();
- const { data, error } = await supabase_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("job_id", scrape1.metadata.scrapeId!)
- .limit(1);
-
- expect(error).toBeFalsy();
- expect(data).toHaveLength(1);
-
- if (data && data.length === 1) {
- const record = data[0];
- expect(record.url).not.toContain("://"); // no url stored
- expect(record.docs).toBeNull();
- expect(record.page_options).toBeNull();
- expect(record.crawler_options).toBeNull();
- }
+ await expectJobRecordIsCleanedUp(scrape1.metadata.scrapeId!);
if (scope === "Request-scoped") {
const status = await scrapeStatusRaw(
@@ -142,55 +88,18 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
expect(postLogs).toHaveLength(0);
- const { data, error } = await supabase_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("job_id", crawl1.id)
- .limit(1);
-
- expect(error).toBeFalsy();
- expect(data).toHaveLength(1);
+ await expectJobRecordIsCleanedUp(crawl1.id);
- if (data && data.length === 1) {
- const record = data[0];
- expect(record.url).not.toContain("://"); // no url stored
- expect(record.docs).toBeNull();
- expect(record.page_options).toBeNull();
- expect(record.crawler_options).toBeNull();
- }
-
- const { data: jobs, error: jobsError } = await supabase_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("crawl_id", crawl1.id);
-
- expect(jobsError).toBeFalsy();
- expect((jobs ?? []).length).toBeGreaterThanOrEqual(1);
-
- for (const job of jobs ?? []) {
- expect(job.url).not.toContain("://"); // no url stored
- expect(job.docs).toBeNull();
- expect(job.page_options).toBeNull();
- expect(job.crawler_options).toBeNull();
- expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry
-
- if (job.success) {
- const gcsJob = await getJobFromGCS(job.job_id);
- expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry
- }
- }
+ const jobs = await expectJobsArrayIsCleanedUp(crawl1.id);
await zdrcleaner(identity.teamId!);
- for (const job of jobs ?? []) {
- const gcsJob = await getJobFromGCS(job.job_id);
- expect(gcsJob).toBeNull();
-
- if (scope === "Request-scoped") {
- const status = await scrapeStatusRaw(job.job_id, identity);
- expect(status.statusCode).toBe(404);
- }
- }
+ await expectJobsAreFullyCleanedAfterZDRCleaner(
+ jobs,
+ scope,
+ identity,
+ scrapeStatusRaw,
+ );
}, 600000);
it("should clean up a batch scrape", async () => {
@@ -225,55 +134,18 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
expect(postLogs).toHaveLength(0);
- const { data, error } = await supabase_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("job_id", crawl1.id)
- .limit(1);
-
- expect(error).toBeFalsy();
- expect(data).toHaveLength(1);
-
- if (data && data.length === 1) {
- const record = data[0];
- expect(record.url).not.toContain("://"); // no url stored
- expect(record.docs).toBeNull();
- expect(record.page_options).toBeNull();
- expect(record.crawler_options).toBeNull();
- }
-
- const { data: jobs, error: jobsError } = await supabase_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("crawl_id", crawl1.id);
+ await expectJobRecordIsCleanedUp(crawl1.id);
- expect(jobsError).toBeFalsy();
- expect((jobs ?? []).length).toBe(2);
-
- for (const job of jobs ?? []) {
- expect(job.url).not.toContain("://"); // no url stored
- expect(job.docs).toBeNull();
- expect(job.page_options).toBeNull();
- expect(job.crawler_options).toBeNull();
- expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry
-
- if (job.success) {
- const gcsJob = await getJobFromGCS(job.job_id);
- expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry
- }
- }
+ const jobs = await expectJobsArrayIsCleanedUp(crawl1.id, 2);
await zdrcleaner(identity.teamId!);
- for (const job of jobs ?? []) {
- const gcsJob = await getJobFromGCS(job.job_id);
- expect(gcsJob).toBeNull();
-
- if (scope === "Request-scoped") {
- const status = await scrapeStatusRaw(job.job_id, identity);
- expect(status.statusCode).toBe(404);
- }
- }
+ await expectJobsAreFullyCleanedAfterZDRCleaner(
+ jobs,
+ scope,
+ identity,
+ scrapeStatusRaw,
+ );
}, 600000);
});
});
diff --git a/apps/api/src/__tests__/snips/zdr-helpers.ts b/apps/api/src/__tests__/snips/zdr-helpers.ts
new file mode 100644
index 000000000..9e1c18b40
--- /dev/null
+++ b/apps/api/src/__tests__/snips/zdr-helpers.ts
@@ -0,0 +1,146 @@
+import { readFile, stat } from "node:fs/promises";
+import { supabase_service } from "../../services/supabase";
+import { getJobFromGCS } from "../../lib/gcs-jobs";
+
+type Identity = { apiKey: string; teamId: string };
+type ScrapeStatusRawFn = (
+ jobId: string,
+ identity: Identity,
+) => Promise<{ statusCode: number }>;
+
+export const logIgnoreList = [
+ "Billing queue created",
+ "No billing operations to process in batch",
+ "billing batch queue",
+ "billing batch processing lock",
+ "Batch billing team",
+ "Successfully billed team",
+ "Billing batch processing",
+ "Processing batch of",
+ "Billing team",
+ "No jobs to process",
+ "nuqHealthCheck metrics",
+ "nuqGetJobToProcess metrics",
+ "Domain frequency processor",
+ "billing operation to batch queue",
+ "billing operation to queue",
+ "billing operation for team",
+ "Added billing operation to queue",
+ "Index RF inserter found",
+ "Redis connected",
+ "Prefetched jobs",
+ "nuqPrefetchJobs metrics",
+ "request completed",
+ "nuqAddJobs metrics",
+ "nuqGetJobs metrics",
+];
+
+export async function getLogs() {
+ const winstonLogFiles = ["firecrawl-app.log", "firecrawl-worker.log"];
+ const existingLogFiles: string[] = [];
+
+ for (const file of winstonLogFiles) {
+ try {
+ await stat(file);
+ existingLogFiles.push(file);
+ } catch {
+ continue;
+ }
+ }
+
+ if (existingLogFiles.length === 0) {
+ console.warn(
+ "No log file found (checked firecrawl-app.log, firecrawl-worker.log)",
+ );
+ return [];
+ }
+
+ const allLogs = await Promise.all(
+ existingLogFiles.map(file => readFile(file, "utf8")),
+ );
+
+ return allLogs
+ .join("\n")
+ .split("\n")
+
+ .map(line => {
+ try {
+ const logEntry = JSON.parse(line);
+ return logEntry.message || line;
+ } catch {
+ return line;
+ }
+ })
+ .filter(
+ x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y)),
+ );
+}
+
+export async function expectJobRecordIsCleanedUp(jobId: string) {
+ const { data, error } = await supabase_service
+ .from("firecrawl_jobs")
+ .select("*")
+ .eq("job_id", jobId)
+ .limit(1);
+
+ expect(error).toBeFalsy();
+ expect(data).toHaveLength(1);
+
+ if (data && data.length === 1) {
+ const record = data[0];
+ expect(record.url).not.toContain("://"); // no url stored
+ expect(record.docs).toBeNull();
+ expect(record.page_options).toBeNull();
+ expect(record.crawler_options).toBeNull();
+ }
+}
+
+export async function expectJobsArrayIsCleanedUp(
+ crawlId: string,
+ expectedJobCount?: number,
+) {
+ const { data: jobs, error: jobsError } = await supabase_service
+ .from("firecrawl_jobs")
+ .select("*")
+ .eq("crawl_id", crawlId);
+
+ expect(jobsError).toBeFalsy();
+
+ if (expectedJobCount !== undefined) {
+ expect((jobs ?? []).length).toBe(expectedJobCount);
+ } else {
+ expect((jobs ?? []).length).toBeGreaterThanOrEqual(1);
+ }
+
+ for (const job of jobs ?? []) {
+ expect(job.url).not.toContain("://"); // no url stored
+ expect(job.docs).toBeNull();
+ expect(job.page_options).toBeNull();
+ expect(job.crawler_options).toBeNull();
+ expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry
+
+ if (job.success) {
+ const gcsJob = await getJobFromGCS(job.job_id);
+ expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry
+ }
+ }
+
+ return jobs ?? [];
+}
+
+export async function expectJobsAreFullyCleanedAfterZDRCleaner(
+ jobs: any[],
+ scope: "Team-scoped" | "Request-scoped",
+ identity: Identity,
+ scrapeStatusRaw: ScrapeStatusRawFn,
+) {
+ for (const job of jobs) {
+ const gcsJob = await getJobFromGCS(job.job_id);
+ expect(gcsJob).toBeNull();
+
+ if (scope === "Request-scoped") {
+ const status = await scrapeStatusRaw(job.job_id, identity);
+ expect(status.statusCode).toBe(404);
+ }
+ }
+}
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index c5ea958b0..57107d350 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -3,7 +3,6 @@ import { type ChildProcess, spawn } from "child_process";
import * as net from "net";
import { basename, join } from "path";
import { HTML_TO_MARKDOWN_PATH } from "./natives";
-import { createWriteStream } from "fs";
const childProcesses = new Set<ChildProcess>();
const stopping = new WeakSet<ChildProcess>(); // processes we're intentionally stopping
@@ -88,8 +87,6 @@ function formatDuration(nanoseconds: bigint): string {
return `${minutes}m ${remainingSeconds.toFixed(0)}s`;
}
-const stream = createWriteStream("firecrawl.log");
-
const PORT = process.env.PORT ?? "3002";
const WORKER_PORT = process.env.WORKER_PORT ?? "3005";
const EXTRACT_WORKER_PORT = process.env.EXTRACT_WORKER_PORT ?? "3004";
@@ -144,7 +141,6 @@ const logger = {
const label = `${color}${colors.bold}${name.padEnd(14)}${colors.reset}`;
console.log(`${label} ${line}`);
}
- stream.write(`${name.padEnd(14)} ${line}\n`);
},
};
diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.ts
index 10a1e11a5..9c61cb9a2 100644
--- a/apps/api/src/lib/logger.ts
+++ b/apps/api/src/lib/logger.ts
@@ -59,13 +59,14 @@ export const logger = winston.createLogger({
filename:
"firecrawl-" +
(process.argv[1].includes("worker") ? "worker" : "app") +
- "-" +
- crypto.randomUUID() +
".log",
format: winston.format.combine(
zeroDataRetentionFilter,
winston.format.json(),
),
+ maxsize: 10 * 1024 * 1024,
+ maxFiles: 3,
+ tailable: true,
}),
]
: []),
diff --git a/docker-compose.yaml b/docker-compose.yaml
index f4da1fcaf..221fc9fb5 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -14,6 +14,12 @@ x-common-service: &common-service
- backend
extra_hosts:
- "host.docker.internal:host-gateway"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
+ compress: "true"
x-common-env: &common-env
REDIS_URL: ${REDIS_URL:-redis://redis:6379}
@@ -57,6 +63,14 @@ services:
BLOCK_MEDIA: ${BLOCK_MEDIA}
networks:
- backend
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
+ compress: "true"
+ tmpfs:
+ - /tmp/.cache:noexec,nosuid,size=1g
api:
<<: *common-service
@@ -84,6 +98,12 @@ services:
networks:
- backend
command: redis-server --bind 0.0.0.0
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "5m"
+ max-file: "2"
+ compress: "true"
nuq-postgres:
build: apps/nuq-postgres
@@ -95,6 +115,12 @@ services:
- backend
ports:
- "5432:5432"
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
+ compress: "true"
networks:
backend: