Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Optimize Logs File Size (#2456)
Abimael Martell784d1063d453
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex bce12dbc7..63a209d2e 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -44,6 +44,7 @@ jobs:NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgresUSE_GO_MARKDOWN_PARSER: trueALLOW_LOCAL_WEBHOOKS: true+ FIRECRAWL_LOG_TO_FILE: truesteps:- uses: actions/checkout@v5@@ -201,7 +202,8 @@ jobs:if: always()run: |mkdir -p logs- cp ./apps/api/firecrawl.log logs/firecrawl.log+ cp ./apps/api/firecrawl-app.log logs/firecrawl.log || true+ cp ./apps/api/firecrawl-worker.log logs/firecrawl-worker.log || true- name: Copy SearXNG logsif: always() && matrix.search == 'searxng'diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.tsindex 9bbb17c47..db853250c 100644--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts@@ -1,5 +1,4 @@import { getJobFromGCS } from "../../../lib/gcs-jobs";-import { supabase_service } from "../../../services/supabase";import {scrape,crawl,@@ -8,51 +7,13 @@ import {zdrcleaner,idmux,} from "./lib";-import { readFile, stat } from "node:fs/promises";import { describeIf, TEST_PRODUCTION } from "../lib";--const logIgnoreList = [- "Billing queue created",- "No billing operations to process in batch",- "billing batch queue",- "billing batch processing lock",- "Batch billing team",- "Successfully billed team",- "Billing batch processing",- "Processing batch of",- "Billing team",- "No jobs to process",- "nuqHealthCheck metrics",- "nuqGetJobToProcess metrics",- "Domain frequency processor",- "billing operation to batch queue",- "billing operation to queue",- "billing operation for team",- "Added billing operation to queue",- "Index RF inserter found",- "Redis connected",- "Prefetched jobs",- "nuqPrefetchJobs metrics",- "request completed",- "nuqAddJobs metrics",- "nuqGetJobs metrics",-];--async function getLogs() {- let logs: string;- try {- await stat("firecrawl.log");- } catch (e) {- console.warn("No firecrawl.log file found");- return [];- }- logs = await readFile("firecrawl.log", "utf8");- return logs- .split("\n")- .filter(- x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y)),- );-}+import {+ getLogs,+ expectJobRecordIsCleanedUp,+ expectJobsArrayIsCleanedUp,+ expectJobsAreFullyCleanedAfterZDRCleaner,+} from "../zdr-helpers";describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {describe.each(["Team-scoped", "Request-scoped"] as const)("%s", scope => {@@ -82,22 +43,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!);expect(gcsJob).toBeNull();- const { data, error } = await supabase_service- .from("firecrawl_jobs")- .select("*")- .eq("job_id", scrape1.metadata.scrapeId!)- .limit(1);-- expect(error).toBeFalsy();- expect(data).toHaveLength(1);-- if (data && data.length === 1) {- const record = data[0];- expect(record.url).not.toContain("://"); // no url stored- expect(record.docs).toBeNull();- expect(record.page_options).toBeNull();- expect(record.crawler_options).toBeNull();- }+ await expectJobRecordIsCleanedUp(scrape1.metadata.scrapeId!);if (scope === "Request-scoped") {const status = await scrapeStatusRaw(@@ -144,55 +90,18 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {expect(postLogs).toHaveLength(0);- const { data, error } = await supabase_service- .from("firecrawl_jobs")- .select("*")- .eq("job_id", crawl1.id)- .limit(1);-- expect(error).toBeFalsy();- expect(data).toHaveLength(1);+ await expectJobRecordIsCleanedUp(crawl1.id);- if (data && data.length === 1) {- const record = data[0];- expect(record.url).not.toContain("://"); // no url stored- expect(record.docs).toBeNull();- expect(record.page_options).toBeNull();- expect(record.crawler_options).toBeNull();- }-- const { data: jobs, error: jobsError } = await supabase_service- .from("firecrawl_jobs")- .select("*")- .eq("crawl_id", crawl1.id);-- expect(jobsError).toBeFalsy();- expect((jobs ?? []).length).toBeGreaterThanOrEqual(1);-- for (const job of jobs ?? []) {- expect(job.url).not.toContain("://"); // no url stored- expect(job.docs).toBeNull();- expect(job.page_options).toBeNull();- expect(job.crawler_options).toBeNull();- expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry-- if (job.success) {- const gcsJob = await getJobFromGCS(job.job_id);- expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry- }- }+ const jobs = await expectJobsArrayIsCleanedUp(crawl1.id);await zdrcleaner(identity.teamId!);- for (const job of jobs ?? []) {- const gcsJob = await getJobFromGCS(job.job_id);- expect(gcsJob).toBeNull();-- if (scope === "Request-scoped") {- const status = await scrapeStatusRaw(job.job_id, identity);- expect(status.statusCode).toBe(404);- }- }+ await expectJobsAreFullyCleanedAfterZDRCleaner(+ jobs,+ scope,+ identity,+ scrapeStatusRaw,+ );}, 600000);it("should clean up a batch scrape", async () => {@@ -229,55 +138,18 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {expect(postLogs).toHaveLength(0);- const { data, error } = await supabase_service- .from("firecrawl_jobs")- .select("*")- .eq("job_id", crawl1.id)- .limit(1);-- expect(error).toBeFalsy();- expect(data).toHaveLength(1);-- if (data && data.length === 1) {- const record = data[0];- expect(record.url).not.toContain("://"); // no url stored- expect(record.docs).toBeNull();- expect(record.page_options).toBeNull();- expect(record.crawler_options).toBeNull();- }-- const { data: jobs, error: jobsError } = await supabase_service- .from("firecrawl_jobs")- .select("*")- .eq("crawl_id", crawl1.id);+ await expectJobRecordIsCleanedUp(crawl1.id);- expect(jobsError).toBeFalsy();- expect((jobs ?? []).length).toBe(2);-- for (const job of jobs ?? []) {- expect(job.url).not.toContain("://"); // no url stored- expect(job.docs).toBeNull();- expect(job.page_options).toBeNull();- expect(job.crawler_options).toBeNull();- expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry-- if (job.success) {- const gcsJob = await getJobFromGCS(job.job_id);- expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry- }- }+ const jobs = await expectJobsArrayIsCleanedUp(crawl1.id, 2);await zdrcleaner(identity.teamId!);- for (const job of jobs ?? []) {- const gcsJob = await getJobFromGCS(job.job_id);- expect(gcsJob).toBeNull();-- if (scope === "Request-scoped") {- const status = await scrapeStatusRaw(job.job_id, identity);- expect(status.statusCode).toBe(404);- }- }+ await expectJobsAreFullyCleanedAfterZDRCleaner(+ jobs,+ scope,+ identity,+ scrapeStatusRaw,+ );}, 600000);});});diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.tsindex 3687a66ad..dcccbe48a 100644--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts@@ -1,4 +1,3 @@-import { supabase_service } from "../../../services/supabase";import { getJobFromGCS } from "../../../lib/gcs-jobs";import {scrape,@@ -8,51 +7,13 @@ import {zdrcleaner,idmux,} from "./lib";-import { readFile, stat } from "node:fs/promises";import { describeIf, TEST_PRODUCTION } from "../lib";--const logIgnoreList = [- "Billing queue created",- "No billing operations to process in batch",- "billing batch queue",- "billing batch processing lock",- "Batch billing team",- "Successfully billed team",- "Billing batch processing",- "Processing batch of",- "Billing team",- "No jobs to process",- "nuqHealthCheck metrics",- "nuqGetJobToProcess metrics",- "Domain frequency processor",- "billing operation to batch queue",- "billing operation to queue",- "billing operation for team",- "Added billing operation to queue",- "Index RF inserter found",- "Redis connected",- "Prefetched jobs",- "nuqPrefetchJobs metrics",- "request completed",- "nuqAddJobs metrics",- "nuqGetJobs metrics",-];--async function getLogs() {- let logs: string;- try {- await stat("firecrawl.log");- } catch (e) {- console.warn("No firecrawl.log file found");- return [];- }- logs = await readFile("firecrawl.log", "utf8");- return logs- .split("\n")- .filter(- x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y)),- );-}+import {+ getLogs,+ expectJobRecordIsCleanedUp,+ expectJobsArrayIsCleanedUp,+ expectJobsAreFullyCleanedAfterZDRCleaner,+} from "../zdr-helpers";describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {describe.each(["Team-scoped", "Request-scoped"] as const)("%s", scope => {@@ -82,22 +43,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!);expect(gcsJob).toBeNull();- const { data, error } = await supabase_service- .from("firecrawl_jobs")- .select("*")- .eq("job_id", scrape1.metadata.scrapeId!)- .limit(1);-- expect(error).toBeFalsy();- expect(data).toHaveLength(1);-- if (data && data.length === 1) {- const record = data[0];- expect(record.url).not.toContain("://"); // no url stored- expect(record.docs).toBeNull();- expect(record.page_options).toBeNull();- expect(record.crawler_options).toBeNull();- }+ await expectJobRecordIsCleanedUp(scrape1.metadata.scrapeId!);if (scope === "Request-scoped") {const status = await scrapeStatusRaw(@@ -142,55 +88,18 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {expect(postLogs).toHaveLength(0);- const { data, error } = await supabase_service- .from("firecrawl_jobs")- .select("*")- .eq("job_id", crawl1.id)- .limit(1);-- expect(error).toBeFalsy();- expect(data).toHaveLength(1);+ await expectJobRecordIsCleanedUp(crawl1.id);- if (data && data.length === 1) {- const record = data[0];- expect(record.url).not.toContain("://"); // no url stored- expect(record.docs).toBeNull();- expect(record.page_options).toBeNull();- expect(record.crawler_options).toBeNull();- }-- const { data: jobs, error: jobsError } = await supabase_service- .from("firecrawl_jobs")- .select("*")- .eq("crawl_id", crawl1.id);-- expect(jobsError).toBeFalsy();- expect((jobs ?? []).length).toBeGreaterThanOrEqual(1);-- for (const job of jobs ?? []) {- expect(job.url).not.toContain("://"); // no url stored- expect(job.docs).toBeNull();- expect(job.page_options).toBeNull();- expect(job.crawler_options).toBeNull();- expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry-- if (job.success) {- const gcsJob = await getJobFromGCS(job.job_id);- expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry- }- }+ const jobs = await expectJobsArrayIsCleanedUp(crawl1.id);await zdrcleaner(identity.teamId!);- for (const job of jobs ?? []) {- const gcsJob = await getJobFromGCS(job.job_id);- expect(gcsJob).toBeNull();-- if (scope === "Request-scoped") {- const status = await scrapeStatusRaw(job.job_id, identity);- expect(status.statusCode).toBe(404);- }- }+ await expectJobsAreFullyCleanedAfterZDRCleaner(+ jobs,+ scope,+ identity,+ scrapeStatusRaw,+ );}, 600000);it("should clean up a batch scrape", async () => {@@ -225,55 +134,18 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {expect(postLogs).toHaveLength(0);- const { data, error } = await supabase_service- .from("firecrawl_jobs")- .select("*")- .eq("job_id", crawl1.id)- .limit(1);-- expect(error).toBeFalsy();- expect(data).toHaveLength(1);-- if (data && data.length === 1) {- const record = data[0];- expect(record.url).not.toContain("://"); // no url stored- expect(record.docs).toBeNull();- expect(record.page_options).toBeNull();- expect(record.crawler_options).toBeNull();- }-- const { data: jobs, error: jobsError } = await supabase_service- .from("firecrawl_jobs")- .select("*")- .eq("crawl_id", crawl1.id);+ await expectJobRecordIsCleanedUp(crawl1.id);- expect(jobsError).toBeFalsy();- expect((jobs ?? []).length).toBe(2);-- for (const job of jobs ?? []) {- expect(job.url).not.toContain("://"); // no url stored- expect(job.docs).toBeNull();- expect(job.page_options).toBeNull();- expect(job.crawler_options).toBeNull();- expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry-- if (job.success) {- const gcsJob = await getJobFromGCS(job.job_id);- expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry- }- }+ const jobs = await expectJobsArrayIsCleanedUp(crawl1.id, 2);await zdrcleaner(identity.teamId!);- for (const job of jobs ?? []) {- const gcsJob = await getJobFromGCS(job.job_id);- expect(gcsJob).toBeNull();-- if (scope === "Request-scoped") {- const status = await scrapeStatusRaw(job.job_id, identity);- expect(status.statusCode).toBe(404);- }- }+ await expectJobsAreFullyCleanedAfterZDRCleaner(+ jobs,+ scope,+ identity,+ scrapeStatusRaw,+ );}, 600000);});});diff --git a/apps/api/src/__tests__/snips/zdr-helpers.ts b/apps/api/src/__tests__/snips/zdr-helpers.tsnew file mode 100644index 000000000..9e1c18b40--- /dev/null+++ b/apps/api/src/__tests__/snips/zdr-helpers.ts@@ -0,0 +1,146 @@+import { readFile, stat } from "node:fs/promises";+import { supabase_service } from "../../services/supabase";+import { getJobFromGCS } from "../../lib/gcs-jobs";++type Identity = { apiKey: string; teamId: string };+type ScrapeStatusRawFn = (+ jobId: string,+ identity: Identity,+) => Promise<{ statusCode: number }>;++export const logIgnoreList = [+ "Billing queue created",+ "No billing operations to process in batch",+ "billing batch queue",+ "billing batch processing lock",+ "Batch billing team",+ "Successfully billed team",+ "Billing batch processing",+ "Processing batch of",+ "Billing team",+ "No jobs to process",+ "nuqHealthCheck metrics",+ "nuqGetJobToProcess metrics",+ "Domain frequency processor",+ "billing operation to batch queue",+ "billing operation to queue",+ "billing operation for team",+ "Added billing operation to queue",+ "Index RF inserter found",+ "Redis connected",+ "Prefetched jobs",+ "nuqPrefetchJobs metrics",+ "request completed",+ "nuqAddJobs metrics",+ "nuqGetJobs metrics",+];++export async function getLogs() {+ const winstonLogFiles = ["firecrawl-app.log", "firecrawl-worker.log"];+ const existingLogFiles: string[] = [];++ for (const file of winstonLogFiles) {+ try {+ await stat(file);+ existingLogFiles.push(file);+ } catch {+ continue;+ }+ }++ if (existingLogFiles.length === 0) {+ console.warn(+ "No log file found (checked firecrawl-app.log, firecrawl-worker.log)",+ );+ return [];+ }++ const allLogs = await Promise.all(+ existingLogFiles.map(file => readFile(file, "utf8")),+ );++ return allLogs+ .join("\n")+ .split("\n")++ .map(line => {+ try {+ const logEntry = JSON.parse(line);+ return logEntry.message || line;+ } catch {+ return line;+ }+ })+ .filter(+ x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y)),+ );+}++export async function expectJobRecordIsCleanedUp(jobId: string) {+ const { data, error } = await supabase_service+ .from("firecrawl_jobs")+ .select("*")+ .eq("job_id", jobId)+ .limit(1);++ expect(error).toBeFalsy();+ expect(data).toHaveLength(1);++ if (data && data.length === 1) {+ const record = data[0];+ expect(record.url).not.toContain("://"); // no url stored+ expect(record.docs).toBeNull();+ expect(record.page_options).toBeNull();+ expect(record.crawler_options).toBeNull();+ }+}++export async function expectJobsArrayIsCleanedUp(+ crawlId: string,+ expectedJobCount?: number,+) {+ const { data: jobs, error: jobsError } = await supabase_service+ .from("firecrawl_jobs")+ .select("*")+ .eq("crawl_id", crawlId);++ expect(jobsError).toBeFalsy();++ if (expectedJobCount !== undefined) {+ expect((jobs ?? []).length).toBe(expectedJobCount);+ } else {+ expect((jobs ?? []).length).toBeGreaterThanOrEqual(1);+ }++ for (const job of jobs ?? []) {+ expect(job.url).not.toContain("://"); // no url stored+ expect(job.docs).toBeNull();+ expect(job.page_options).toBeNull();+ expect(job.crawler_options).toBeNull();+ expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry++ if (job.success) {+ const gcsJob = await getJobFromGCS(job.job_id);+ expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry+ }+ }++ return jobs ?? [];+}++export async function expectJobsAreFullyCleanedAfterZDRCleaner(+ jobs: any[],+ scope: "Team-scoped" | "Request-scoped",+ identity: Identity,+ scrapeStatusRaw: ScrapeStatusRawFn,+) {+ for (const job of jobs) {+ const gcsJob = await getJobFromGCS(job.job_id);+ expect(gcsJob).toBeNull();++ if (scope === "Request-scoped") {+ const status = await scrapeStatusRaw(job.job_id, identity);+ expect(status.statusCode).toBe(404);+ }+ }+}diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex c5ea958b0..57107d350 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -3,7 +3,6 @@ import { type ChildProcess, spawn } from "child_process";import * as net from "net";import { basename, join } from "path";import { HTML_TO_MARKDOWN_PATH } from "./natives";-import { createWriteStream } from "fs";const childProcesses = new Set<ChildProcess>();const stopping = new WeakSet<ChildProcess>(); // processes we're intentionally stopping@@ -88,8 +87,6 @@ function formatDuration(nanoseconds: bigint): string {return `${minutes}m ${remainingSeconds.toFixed(0)}s`;}-const stream = createWriteStream("firecrawl.log");-const PORT = process.env.PORT ?? "3002";const WORKER_PORT = process.env.WORKER_PORT ?? "3005";const EXTRACT_WORKER_PORT = process.env.EXTRACT_WORKER_PORT ?? "3004";@@ -144,7 +141,6 @@ const logger = {const label = `${color}${colors.bold}${name.padEnd(14)}${colors.reset}`;console.log(`${label} ${line}`);}- stream.write(`${name.padEnd(14)} ${line}\n`);},};diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.tsindex 10a1e11a5..9c61cb9a2 100644--- a/apps/api/src/lib/logger.ts+++ b/apps/api/src/lib/logger.ts@@ -59,13 +59,14 @@ export const logger = winston.createLogger({filename:"firecrawl-" +(process.argv[1].includes("worker") ? "worker" : "app") +- "-" +- crypto.randomUUID() +".log",format: winston.format.combine(zeroDataRetentionFilter,winston.format.json(),),+ maxsize: 10 * 1024 * 1024,+ maxFiles: 3,+ tailable: true,}),]: []),diff --git a/docker-compose.yaml b/docker-compose.yamlindex f4da1fcaf..221fc9fb5 100644--- a/docker-compose.yaml+++ b/docker-compose.yaml@@ -14,6 +14,12 @@ x-common-service: &common-service- backendextra_hosts:- "host.docker.internal:host-gateway"+ logging:+ driver: "json-file"+ options:+ max-size: "10m"+ max-file: "3"+ compress: "true"x-common-env: &common-envREDIS_URL: ${REDIS_URL:-redis://redis:6379}@@ -57,6 +63,14 @@ services:BLOCK_MEDIA: ${BLOCK_MEDIA}networks:- backend+ logging:+ driver: "json-file"+ options:+ max-size: "10m"+ max-file: "3"+ compress: "true"+ tmpfs:+ - /tmp/.cache:noexec,nosuid,size=1gapi:<<: *common-service@@ -84,6 +98,12 @@ services:networks:- backendcommand: redis-server --bind 0.0.0.0+ logging:+ driver: "json-file"+ options:+ max-size: "5m"+ max-file: "2"+ compress: "true"nuq-postgres:build: apps/nuq-postgres@@ -95,6 +115,12 @@ services:- backendports:- "5432:5432"+ logging:+ driver: "json-file"+ options:+ max-size: "10m"+ max-file: "3"+ compress: "true"networks:backend: