Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update Dockerfile (#1231) (#1232)
Gergő Móricz76e1f29ae8e1
diff --git a/apps/api/Dockerfile b/apps/api/Dockerfileindex 051bb3574..5818b886e 100644--- a/apps/api/Dockerfile+++ b/apps/api/Dockerfile@@ -1,11 +1,7 @@-FROM node:20-slim AS base+FROM node:22-slim AS baseENV PNPM_HOME="/pnpm"ENV PATH="$PNPM_HOME:$PATH"-LABEL fly_launch_runtime="Node.js"--#temp solution to https://github.com/nodejs/corepack/issues/612-RUN npm i -g corepack@latestRUN corepack enableCOPY . /app@@ -15,14 +11,10 @@ FROM base AS prod-depsRUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfileFROM base AS build-RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile--RUN apt-get clean && apt-get update -qq && apt-get install -y ca-certificates && update-ca-certificates-RUN pnpm install-RUN pnpm run build+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install && pnpm run build# Install Go-FROM golang:1.19 AS go-base+FROM golang:1.24 AS go-baseCOPY sharedLibs/go-html-to-md /app/sharedLibs/go-html-to-md# Install Go dependencies and build parser lib@@ -32,23 +24,22 @@ RUN cd /app/sharedLibs/go-html-to-md && \chmod +x html-to-markdown.so# Install Rust-FROM rust:1-bullseye AS rust-base+FROM rust:1-slim AS rust-baseCOPY sharedLibs/html-transformer /app/sharedLibs/html-transformer-# Install Go dependencies and build parser lib+# Install Rust dependencies and build transformer libRUN cd /app/sharedLibs/html-transformer && \cargo build --release && \chmod +x target/release/libhtml_transformer.soFROM base+COPY --from=build /app/dist /app/distCOPY --from=prod-deps /app/node_modules /app/node_modules-COPY --from=build /app /appCOPY --from=go-base /app/sharedLibs/go-html-to-md/html-to-markdown.so /app/sharedLibs/go-html-to-md/html-to-markdown.soCOPY --from=rust-base /app/sharedLibs/html-transformer/target/release/libhtml_transformer.so /app/sharedLibs/html-transformer/target/release/libhtml_transformer.so# Start the server by default, this can be overwritten at runtimeEXPOSE 8080-ENV PUPPETEER_EXECUTABLE_PATH="/usr/bin/chromium"# Make sure the entrypoint script has the correct line endingsRUN sed -i 's/\r$//' /app/docker-entrypoint.shdiff --git a/apps/api/package.json b/apps/api/package.jsonindex 398e3398a..745783b8e 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -30,7 +30,6 @@"author": "","license": "ISC","devDependencies": {- "@flydotio/dockerfile": "^0.4.10","@jest/globals": "^29.7.0","@tsconfig/recommended": "^1.0.3","@types/body-parser": "^1.19.2",@@ -42,14 +41,10 @@"@types/node": "^20.14.1","@types/pdf-parse": "^1.1.4","@types/supertest": "^6.0.2",- "body-parser": "^1.20.1",- "express": "^4.18.2","jest": "^29.6.3","jest-fetch-mock": "^3.0.3",- "mammoth": "^1.7.2","nodemon": "^2.0.20","prettier": "^3.4.2",- "supabase": "^1.77.9","supertest": "^6.3.3","ts-jest": "^29.1.1","ts-node": "^10.9.1",@@ -78,6 +73,7 @@"async-mutex": "^0.5.0","axios": "^1.3.4","axios-retry": "^4.5.0",+ "body-parser": "^1.20.1","bottleneck": "^2.19.5","bullmq": "^5.36.0","cacheable-lookup": "^6.1.0",@@ -90,6 +86,7 @@"dotenv": "^16.3.1","dotenv-cli": "^7.4.2","escape-html": "^1.0.3",+ "express": "^4.18.2","express-rate-limit": "^7.3.1","express-ws": "^5.0.2","glob": "^10.4.2",@@ -106,6 +103,7 @@"lodash": "^4.17.21","logsnag": "^1.0.0","luxon": "^3.4.3",+ "mammoth": "^1.7.2","marked": "^14.1.2","md5": "^2.3.0","moment": "^2.29.4",@@ -123,6 +121,7 @@"robots-parser": "^3.0.1","scrapingbee": "^1.7.4","stripe": "^16.1.0",+ "supabase": "^1.77.9","systeminformation": "^5.22.11","tldts": "^6.1.75","turndown": "^7.1.3",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 8ed0415f0..b11784a39 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -74,6 +74,9 @@ importers:axios-retry:specifier: ^4.5.0version: 4.5.0([email protected])+ body-parser:+ specifier: ^1.20.1+ version: 1.20.2bottleneck:specifier: ^2.19.5version: 2.19.5@@ -110,6 +113,9 @@ importers:escape-html:specifier: ^1.0.3version: 1.0.3+ express:+ specifier: ^4.18.2+ version: 4.19.2express-rate-limit:specifier: ^7.3.1version: 7.3.1([email protected])@@ -158,6 +164,9 @@ importers:luxon:specifier: ^3.4.3version: 3.4.4+ mammoth:+ specifier: ^1.7.2+ version: 1.7.2marked:specifier: ^14.1.2version: 14.1.2@@ -209,6 +218,9 @@ importers:stripe:specifier: ^16.1.0version: 16.1.0+ supabase:+ specifier: ^1.77.9+ version: 1.172.2systeminformation:specifier: ^5.22.11version: 5.22.11@@ -252,9 +264,6 @@ importers:specifier: ^3.24.2version: 3.24.2devDependencies:- '@flydotio/dockerfile':- specifier: ^0.4.10- version: 0.4.11'@jest/globals':specifier: ^29.7.0version: 29.7.0@@ -288,30 +297,18 @@ importers:'@types/supertest':specifier: ^6.0.2version: 6.0.2- body-parser:- specifier: ^1.20.1- version: 1.20.2- express:- specifier: ^4.18.2- version: 4.19.2jest:specifier: ^29.6.3version: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))jest-fetch-mock:specifier: ^3.0.3version: 3.0.3([email protected])- mammoth:- specifier: ^1.7.2- version: 1.7.2nodemon:specifier: ^2.0.20version: 2.0.22prettier:specifier: ^3.4.2version: 3.4.2- supabase:- specifier: ^1.77.9- version: 1.172.2supertest:specifier: ^6.3.3version: 6.3.4@@ -773,11 +770,6 @@ packages:'@dqbd/[email protected]':resolution: {integrity: sha512-v2gz0V6DiuR2TsALM32TkBThf6LdjLbxe6HS/nx9/KJxuDX0Z7SGX7N7PvQfqIvRyus42jI9poVUqezc/j/aQw==}- '@flydotio/[email protected]':- resolution: {integrity: sha512-L52UAfrOhmAn3T4TxpeRofQOSO+Kctg+uraB4nLzo4mvvh+4Z7HYxSi7Dnq0Kirz+xx6fDIc4OMNT1EdaORecA==}- engines: {node: '>=16.0.0'}- hasBin: true-'@ioredis/[email protected]':resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}@@ -2369,10 +2361,6 @@ packages:resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==}engines: {node: '>=0.3.1'}- resolution: {integrity: sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==}- engines: {node: '>=0.3.1'}-resolution: {integrity: sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==}@@ -4280,9 +4268,6 @@ packages:resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}engines: {node: '>=8'}- resolution: {integrity: sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==}-resolution: {integrity: sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==}@@ -5730,14 +5715,6 @@ snapshots:'@dqbd/[email protected]': {}- '@flydotio/[email protected]':- dependencies:- chalk: 5.3.0- diff: 5.2.0- ejs: 3.1.10- shell-quote: 1.8.1- yargs: 17.7.2-'@ioredis/[email protected]': {}'@isaacs/[email protected]':@@ -7729,8 +7706,6 @@ snapshots:- [email protected]: {}-@@ -9834,8 +9809,6 @@ snapshots:- [email protected]: {}-diff --git a/docker-compose.yaml b/docker-compose.yamlindex 58008efe0..f238548d6 100644--- a/docker-compose.yaml+++ b/docker-compose.yaml@@ -3,8 +3,8 @@ name: firecrawlx-common-service: &common-service# NOTE: If you want to build the service locally,# uncomment the build: statement and comment out the image: statement- image: ghcr.io/mendableai/firecrawl- # build: apps/api+ # image: ghcr.io/mendableai/firecrawl+ build: apps/apiulimits:nofile: