Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

add initial codeowners

Gergő Móricz115b6b61c453
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 000000000..0cfdd19a0
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1,91 @@
+# Firecrawl Ownership Chart
+
+# api
+
+## V1 API
+/apps/api/src/routes/v1.ts @mogery @nickscamara
+
+### /scrape
+/apps/api/src/controllers/v1/scrape* @mogery
+
+### /crawl
+/apps/api/src/controllers/v1/crawl* @mogery
+
+### /batch/scrape
+/apps/api/src/controllers/v1/batch-scrape* @mogery
+
+### /extract
+/apps/api/src/controllers/v1/extract* @nickscamara
+/apps/api/src/lib/extract/* @nickscamara
+/apps/api/src/lib/generic-ai.ts @mogery @nickscamara # (AI SDK)
+
+### /map
+/apps/api/src/controllers/v1/map* @nickscamara
+/apps/api/src/lib/map-cosine.ts @nickscamara
+
+### /search
+/apps/api/src/controllers/v1/search* @nickscamara
+/apps/api/src/search/* @nickscamara
+
+### /llmstxt
+/apps/api/src/controllers/v1/generate-llmstxt* @ericciarla
+/apps/api/src/lib/generate-llmstxt/* @ericciarla
+
+### /deep-research
+/apps/api/src/controllers/v1/deep-research* @nickscamara
+/apps/api/src/lib/deep-research/* @nickscamara
+
+### Input Validation/Zod
+/apps/api/src/controllers/v1/types.ts @mogery
+
+## V0 API, deprecated
+/apps/api/src/controllers/v0/* @mogery @nickscamara
+/apps/api/src/routes/v0.ts @mogery @nickscamara
+
+# Worker
+
+## scrapeURL
+/apps/api/src/scraper/scrapeURL/* @mogery
+
+### crawler
+/apps/api/src/lib/crawl-redis* @mogery
+
+### remnants of WebScraper/WebCrawler
+/apps/api/src/scraper/WebScraper/* @mogery @nickscamara
+
+## concurrency limits
+/apps/api/src/lib/concurrency-limit.ts @mogery @nickscamara
+
+## BullMQ-related code
+/apps/api/src/services/queue-worker.ts @mogery @nickscamara
+/apps/api/src/main/runWebScraper.ts @mogery @nickscamara
+/apps/api/src/services/queue* @mogery @nickscamara
+/apps/api/src/lib/job-priority.ts @nickscamara @mogery
+
+# Shared Libraries
+/apps/api/sharedLibs/go-html-to-md/* @tomkosm
+/apps/api/src/lib/html-to-markdown.ts @tomkosm
+/apps/api/sharedLibs/html-transformer/* @mogery
+/apps/api/src/lib/html-transformer.ts @mogery
+
+# playwright-serice-ts
+/apps/playwright-service-ts/* @mogery
+
+# self-hosting
+/docker-compose.yaml @mogery
+/SELF_HOST.md @mogery
+
+# SDKs
+/apps/python-sdk/* @rafaelsideguide @nickscamara
+/apps/js-sdk/* @mogery @nickscamara
+/apps/rust-sdk/* @mogery
+/apps/go-sdk/* @rafaelsideguide
+
+# CI/CD and GitHub Workflows
+/.github/* @mogery @rafaelsideguide
+
+# Tests
+/apps/api/src/__tests__/snips/* @mogery
+
+# Examples
+/examples/* @ericciarla @nickscamara
diff --git a/apps/api/fly.staging.toml b/apps/api/fly.staging.toml
deleted file mode 100644
index db1ed1833..000000000
--- a/apps/api/fly.staging.toml
+++ /dev/null
@@ -1,66 +0,0 @@
-# fly.toml app configuration file generated for firecrawl-scraper-js on 2024-04-07T21:09:59-03:00
-#
-# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
-#
-
-app = 'staging-firecrawl-scraper-js'
-primary_region = 'mia'
-kill_signal = 'SIGINT'
-kill_timeout = '30s'
-
-[build]
-
-[processes]
- app = 'node dist/src/index.js'
- worker = 'node dist/src/services/queue-worker.js'
-
-[http_service]
- internal_port = 8080
- force_https = true
- auto_stop_machines = true
- auto_start_machines = true
- min_machines_running = 2
- processes = ['app']
-
-[http_service.concurrency]
- type = "requests"
- # hard_limit = 100
- soft_limit = 100
-
-[[http_service.checks]]
- grace_period = "10s"
- interval = "30s"
- method = "GET"
- timeout = "5s"
- path = "/"
-
-
-[[services]]
- protocol = 'tcp'
- internal_port = 8080
- processes = ['worker']
-
-[[services.ports]]
- port = 80
- handlers = ['http']
- force_https = true
-
-[[services.ports]]
- port = 443
- handlers = ['tls', 'http']
-
- [services.concurrency]
- type = 'connections'
- # hard_limit = 25
- soft_limit = 100
-
-[[vm]]
- size = 'performance-2x'
- processes = ['app','worker']
- memory = 8192
-
-
-
-
-
-
diff --git a/apps/api/fly.toml b/apps/api/fly.toml
deleted file mode 100644
index c0c874013..000000000
--- a/apps/api/fly.toml
+++ /dev/null
@@ -1,63 +0,0 @@
-# fly.toml app configuration file generated for firecrawl-scraper-js on 2024-04-07T21:09:59-03:00
-#
-# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
-#
-
-app = 'firecrawl-scraper-js'
-primary_region = 'iad'
-kill_signal = 'SIGINT'
-kill_timeout = '30s'
-
-[build]
-
-[processes]
- app = 'node --max-old-space-size=8192 dist/src/index.js'
- worker = 'node --max-old-space-size=8192 dist/src/services/queue-worker.js'
-
-[http_service]
- internal_port = 8080
- force_https = true
- auto_stop_machines = false
- auto_start_machines = true
- min_machines_running = 2
- processes = ['app']
-
-[http_service.concurrency]
- type = "requests"
- # hard_limit = 200
- soft_limit = 200
-
-[[http_service.checks]]
- grace_period = "20s"
- interval = "30s"
- method = "GET"
- timeout = "15s"
- path = "/"
-
-[[services]]
- protocol = 'tcp'
- internal_port = 8080
- processes = ['app']
-
-[[services.ports]]
- port = 80
- handlers = ['http']
- force_https = true
-
-[[services.ports]]
- port = 443
- handlers = ['tls', 'http']
-
- [services.concurrency]
- type = 'connections'
- # hard_limit = 30
- soft_limit = 200
-
-[[vm]]
- size = 'performance-4x'
- processes = ['app']
-
-
-
-
-
diff --git a/apps/api/utils/find_uncovered_files.sh b/apps/api/utils/find_uncovered_files.sh
new file mode 100755
index 000000000..32204decb
--- /dev/null
+++ b/apps/api/utils/find_uncovered_files.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Get all files tracked by git
+git ls-files > /tmp/all_files.txt
+
+# Get files matched by CODEOWNERS
+while read -r line; do
+ # Skip comments and empty lines
+ [[ "$line" =~ ^#.*$ ]] && continue
+ [[ -z "$line" ]] && continue
+
+ # Extract the path pattern
+ pattern=$(echo "$line" | awk '{print $1}')
+
+ # Convert the pattern to a form git understands
+ # Remove leading slash if present
+ pattern=${pattern#/}
+
+ # List files matching this pattern
+ git ls-files "$pattern" 2>/dev/null >> /tmp/covered_files.txt
+done < .github/CODEOWNERS
+
+# Sort and get unique entries
+sort -u /tmp/covered_files.txt > /tmp/covered_files_unique.txt
+
+# Find files that are in all_files but not in covered_files
+comm -23 /tmp/all_files.txt /tmp/covered_files_unique.txt
+
+# Cleanup
+rm /tmp/all_files.txt /tmp/covered_files.txt /tmp/covered_files_unique.txt