Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): answer unmatched and wrong-method paths with JSON (#4552)
Himadri Mishraf31884ae57f9
diff --git a/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts b/apps/api/src/__tests__/routes/not-found-metrics.routes.test.tsnew file mode 100644index 000000000..1217c8ace--- /dev/null+++ b/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts@@ -0,0 +1,144 @@+import express from "express";+import request from "supertest";++vi.mock("../../services/autumn/autumn.service", () => ({+ autumnService: { checkCredits: vi.fn() },+ CREDITS_FEATURE_ID: "CREDITS",+}));+vi.mock("../../services/autumn/usage", () => ({+ getTeamBalance: vi.fn(),+}));+vi.mock("../../controllers/auth", () => ({+ authenticateUser: vi.fn(),+}));+vi.mock("../../services/idempotency/create", () => ({+ createIdempotencyKey: vi.fn(),+}));+vi.mock("../../services/idempotency/validate", () => ({+ validateIdempotencyKey: vi.fn(),+}));+vi.mock("geoip-country", () => ({ lookup: vi.fn(() => null) }));++import { notFoundHandler } from "../../lib/not-found";+import { httpRequestDurationSeconds } from "../../lib/http-metrics";+import { requestTimingMiddleware } from "../../routes/shared";++// The real timing middleware patches res.json, so every JSON answer produced+// after it runs -- including the terminal 404/405 -- is observed by the+// histogram. Unmatched paths must not mint a new time series per URL.+function appUnderTest() {+ const app = express();++ const v2 = express.Router();+ v2.use(requestTimingMiddleware("v2"));+ v2.post("/scrape", (_req, res) => res.status(200).json({ success: true }));+ // Mirrors the real /v2/crawl/:jobId table (src/routes/v2.ts): only GET and+ // DELETE are registered, never PATCH.+ v2.get("/crawl/:jobId", (_req, res) => res.status(200).json({ success: true }));+ v2.delete("/crawl/:jobId", (_req, res) => res.status(200).json({ success: true }));++ app.use("/v2", v2);+ app.use(notFoundHandler);+ return app;+}++async function recordedRoutes(): Promise<string[]> {+ const metric = await httpRequestDurationSeconds.get();+ return metric.values+ .filter(value => value.metricName === "http_request_duration_seconds_count")+ .map(value => String(value.labels.route));+}++describe("not-found handler metric labels", () => {+ beforeEach(() => {+ httpRequestDurationSeconds.reset();+ });++ it("labels unknown paths with a single constant instead of the raw path", async () => {+ const app = appUnderTest();++ await request(app).get("/v2/wp-login.php");+ await request(app).get("/v2/.env");+ await request(app).get("/v2/some/deep/scanner/path");++ const routes = await recordedRoutes();++ expect(routes.length).toBeGreaterThan(0);+ expect(new Set(routes)).toEqual(new Set(["unmatched"]));+ });++ it("labels wrong-method hits on a static path with the same constant", async () => {+ const app = appUnderTest();++ await request(app).get("/v2/scrape");+ await request(app).delete("/v2/scrape");++ const routes = await recordedRoutes();++ expect(routes.length).toBeGreaterThan(0);+ expect(new Set(routes)).toEqual(new Set(["unmatched"]));+ });++ it("labels a wrong-method hit on a genuinely parameterised path with the same constant", async () => {+ const app = appUnderTest();++ // PATCH is not registered for /v2/crawl/:jobId -- only GET and DELETE are.+ const res = await request(app).patch("/v2/crawl/abc-123");++ expect(res.status).toBe(405);+ expect(res.headers["allow"]).toBe("DELETE, GET, HEAD");+ expect(res.body).toEqual({+ success: false,+ code: "METHOD_NOT_ALLOWED",+ error:+ "PATCH /v2/crawl/abc-123 is not supported. Use DELETE, GET, HEAD instead.",+ allowed_methods: ["DELETE", "GET", "HEAD"],+ documentation_url:+ "https://docs.firecrawl.dev/api-reference/introduction",+ });++ const routes = await recordedRoutes();++ expect(routes.length).toBeGreaterThan(0);+ expect(new Set(routes)).toEqual(new Set(["unmatched"]));+ });++ it("leaves the label of a matched route untouched", async () => {+ const app = appUnderTest();++ await request(app).post("/v2/scrape");++ expect(await recordedRoutes()).toEqual(["/scrape"]);+ });+});++describe("terminal not-found handler security headers", () => {+ it("sets nosniff on the JSON 404", async () => {+ const res = await request(appUnderTest()).get("/v2/nonexistent-xyz");++ expect(res.status).toBe(404);+ expect(res.headers["x-content-type-options"]).toBe("nosniff");+ expect(res.body).toEqual({+ success: false,+ code: "NOT_FOUND",+ error: "GET /v2/nonexistent-xyz is not a Firecrawl API endpoint.",+ documentation_url:+ "https://docs.firecrawl.dev/api-reference/introduction",+ });+ });++ it("sets nosniff on the JSON 405", async () => {+ const res = await request(appUnderTest()).get("/v2/scrape");++ expect(res.status).toBe(405);+ expect(res.headers["x-content-type-options"]).toBe("nosniff");+ expect(res.body).toEqual({+ success: false,+ code: "METHOD_NOT_ALLOWED",+ error: "GET /v2/scrape is not supported. Use POST instead.",+ allowed_methods: ["POST"],+ documentation_url:+ "https://docs.firecrawl.dev/api-reference/introduction",+ });+ });+});diff --git a/apps/api/src/__tests__/routes/not-found.routes.test.ts b/apps/api/src/__tests__/routes/not-found.routes.test.tsnew file mode 100644index 000000000..9811a18fb--- /dev/null+++ b/apps/api/src/__tests__/routes/not-found.routes.test.ts@@ -0,0 +1,181 @@+import express from "express";+import request from "supertest";+import { notFoundHandler } from "../../lib/not-found";++// A stand-in for the real app's mount structure: a versioned router mounted+// under a prefix, a sub-router mounted inside it, and a root descriptor. The+// real routers pull in the whole service graph, so the shape is replayed here+// rather than imported.+function appUnderTest() {+ const app = express();++ app.get("/", (_req, res) => {+ res.json({+ message: "Firecrawl API",+ documentation_url: "https://docs.firecrawl.dev",+ });+ });++ const nested = express.Router();+ nested.post("/", (_req, res) => res.status(200).json({ success: true }));+ nested.delete("/", (_req, res) => res.status(200).json({ success: true }));++ const v2 = express.Router();+ v2.post("/scrape", (_req, res) => res.status(200).json({ success: true }));+ v2.post("/search", (_req, res) => res.status(200).json({ success: true }));+ v2.get("/crawl/:id", (_req, res) => res.status(200).json({ success: true }));+ v2.delete("/crawl/:id", (_req, res) =>+ res.status(200).json({ success: true }),+ );+ v2.post("/scrape/:jobId/interact", (_req, res) =>+ res.status(200).json({ success: true }),+ );+ v2.delete("/scrape/:jobId/interact", (_req, res) =>+ res.status(200).json({ success: true }),+ );+ v2.use("/developer", nested);++ // Mounted with no path, the way `v0Router` and `adminRouter` are in+ // src/index.ts. Express flags such a layer as matching everything and never+ // consults its matchers, so the walk has to handle it explicitly.+ const rootMounted = express.Router();+ rootMounted.post("/v0/scrape", (_req, res) =>+ res.status(200).json({ success: true }),+ );+ rootMounted.get("/v0/health/liveness", (_req, res) =>+ res.status(200).json({ success: true }),+ );++ const adminMounted = express.Router();+ adminMounted.post("/admin/:key/queues", (_req, res) =>+ res.status(200).json({ success: true }),+ );++ app.use("/v2", v2);+ app.use(rootMounted);+ app.use(adminMounted);+ app.use(notFoundHandler);+ return app;+}++describe("terminal not-found handler", () => {+ it("answers an unknown path with JSON, not Express's HTML page", async () => {+ const res = await request(appUnderTest()).get("/v2/nonexistent-xyz");++ expect(res.status).toBe(404);+ expect(res.headers["content-type"]).toMatch(/application\/json/);+ expect(res.text).not.toContain("Cannot GET");+ expect(res.body).toEqual({+ success: false,+ code: "NOT_FOUND",+ error: "GET /v2/nonexistent-xyz is not a Firecrawl API endpoint.",+ documentation_url:+ "https://docs.firecrawl.dev/api-reference/introduction",+ });+ });++ it.each(["/v2/scrape", "/v2/search"])(+ "answers GET %s with 405 and an Allow header",+ async path => {+ const res = await request(appUnderTest()).get(path);++ expect(res.status).toBe(405);+ expect(res.headers["content-type"]).toMatch(/application\/json/);+ expect(res.headers["allow"]).toBe("POST");+ expect(res.text).not.toContain("Cannot GET");+ expect(res.body.code).toBe("METHOD_NOT_ALLOWED");+ expect(res.body.allowed_methods).toEqual(["POST"]);+ expect(res.body.documentation_url).toBe(+ "https://docs.firecrawl.dev/api-reference/introduction",+ );+ },+ );++ it("lists every other method for a path that has several", async () => {+ const res = await request(appUnderTest()).post("/v2/crawl/abc");++ expect(res.status).toBe(405);+ expect(res.headers["allow"]).toBe("DELETE, GET, HEAD");+ expect(res.body.allowed_methods).toEqual(["DELETE", "GET", "HEAD"]);+ });++ it("answers GET on the interact path with 405, not Cannot GET", async () => {+ const res = await request(appUnderTest()).get("/v2/scrape/abc/interact");++ expect(res.status).toBe(405);+ expect(res.text).not.toContain("Cannot GET");+ expect(res.headers["allow"]).toBe("DELETE, POST");+ });++ it("resolves paths inside a mounted sub-router", async () => {+ const res = await request(appUnderTest()).get("/v2/developer");++ expect(res.status).toBe(405);+ expect(res.headers["allow"]).toBe("DELETE, POST");+ });++ it("answers GET on a POST-only route in a root-mounted router with 405", async () => {+ const res = await request(appUnderTest()).get("/v0/scrape");++ expect(res.status).toBe(405);+ expect(res.headers["allow"]).toBe("POST");+ expect(res.text).not.toContain("Cannot GET");+ expect(res.body.code).toBe("METHOD_NOT_ALLOWED");+ expect(res.body.allowed_methods).toEqual(["POST"]);+ });++ it("answers POST on a GET-only route in a root-mounted router with 405", async () => {+ const res = await request(appUnderTest()).post("/v0/health/liveness");++ expect(res.status).toBe(405);+ expect(res.headers["allow"]).toBe("GET, HEAD");+ expect(res.body.allowed_methods).toEqual(["GET", "HEAD"]);+ });++ it("resolves admin paths in a second root-mounted router", async () => {+ const res = await request(appUnderTest()).get("/admin/secret/queues");++ expect(res.status).toBe(405);+ expect(res.headers["allow"]).toBe("POST");+ });++ it("still answers an unknown path under a root mount with a JSON 404", async () => {+ const res = await request(appUnderTest()).get("/v0/nonexistent-xyz");++ expect(res.status).toBe(404);+ expect(res.headers["content-type"]).toMatch(/application\/json/);+ expect(res.headers["allow"]).toBeUndefined();+ expect(res.body.code).toBe("NOT_FOUND");+ });++ it("leaves existing routes alone", async () => {+ const app = appUnderTest();++ const scrape = await request(app).post("/v2/scrape");+ expect(scrape.status).toBe(200);+ expect(scrape.body).toEqual({ success: true });++ const status = await request(app).get("/v2/crawl/abc");+ expect(status.status).toBe(200);++ const interact = await request(app).post("/v2/scrape/abc/interact");+ expect(interact.status).toBe(200);++ const developer = await request(app).post("/v2/developer");+ expect(developer.status).toBe(200);++ const root = await request(app).get("/");+ expect(root.status).toBe(200);+ expect(root.body.message).toBe("Firecrawl API");++ const v0Scrape = await request(app).post("/v0/scrape");+ expect(v0Scrape.status).toBe(200);+ expect(v0Scrape.body).toEqual({ success: true });++ const liveness = await request(app).get("/v0/health/liveness");+ expect(liveness.status).toBe(200);++ const admin = await request(app).post("/admin/secret/queues");+ expect(admin.status).toBe(200);+ });+});diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 4c8af690c..8beb321a2 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -45,6 +45,7 @@ import { shutdownWebhookQueue } from "./services/webhook";import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";import { isKeylessConfigured } from "./lib/keyless";import { shutdownPubSubLogging } from "./services/logging/log_job";+import { notFoundHandler } from "./lib/not-found";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");@@ -212,6 +213,10 @@ app.get("/is-production", (req, res) => {res.send({ isProduction: global.isProduction });});+// Terminal handler for unmatched paths. Must stay after every route and before+// the error middleware below, or Express falls back to its default HTML page.+app.use(notFoundHandler);+app.use((err: unknown,diff --git a/apps/api/src/lib/http-metrics.ts b/apps/api/src/lib/http-metrics.tsindex 6e8ee366a..e3808b5cf 100644--- a/apps/api/src/lib/http-metrics.ts+++ b/apps/api/src/lib/http-metrics.ts@@ -11,9 +11,27 @@ export const httpRequestDurationSeconds = new Histogram({buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],});+/**+ * Single `route` label for every request that matched no route. Without it the+ * fallback below would mint a new time series per scanned URL, since unmatched+ * requests reach the terminal handler and answer with `res.json`, which the+ * timing middleware observes.+ */+const UNMATCHED_ROUTE = "unmatched";++const UNMATCHED_FLAG = Symbol.for("firecrawl.httpMetrics.unmatchedRoute");++/** Called by the terminal not-found handler before it answers. */+export function markRouteUnmatched(req: Request): void {+ (req as unknown as Record<symbol, boolean>)[UNMATCHED_FLAG] = true;+}+export function getRoutePattern(req: Request): string {if (req.route?.path) {return req.route.path;}+ if ((req as unknown as Record<symbol, boolean>)[UNMATCHED_FLAG] === true) {+ return UNMATCHED_ROUTE;+ }return req.path.replace(UUID_REGEX, ":id");}diff --git a/apps/api/src/lib/not-found.ts b/apps/api/src/lib/not-found.tsnew file mode 100644index 000000000..4e5a00048--- /dev/null+++ b/apps/api/src/lib/not-found.ts@@ -0,0 +1,138 @@+import type { NextFunction, Request, Response } from "express";+import { markRouteUnmatched } from "./http-metrics";++const NOT_FOUND_DOCUMENTATION_URL =+ "https://docs.firecrawl.dev/api-reference/introduction";++// Express 5 router internals. Layers expose pure matcher functions, so this+// walk never mutates routing state.+type LayerMatch = { path: string } | false;+type Matcher = (path: string) => LayerMatch;+type Layer = {+ matchers?: Matcher[];+ // Set by Express for a router mounted with no path (`app.use(router)`).+ // Express short-circuits on this flag and never consults `matchers`, whose+ // patterns only match "/" for such a layer.+ slash?: boolean;+ route?: { methods?: Record<string, boolean> };+ handle?: unknown;+};+type StackHolder = { stack?: Layer[] };++function hasStack(value: unknown): value is { stack: Layer[] } {+ if (typeof value !== "function" && typeof value !== "object") {+ return false;+ }+ return value !== null && Array.isArray((value as StackHolder).stack);+}++function walk(stack: Layer[], path: string, found: Set<string>): void {+ for (const layer of stack) {+ const matchers = layer.matchers;+ if (!Array.isArray(matchers)) {+ continue;+ }++ // A root-mounted layer matches every path and consumes no prefix, so the+ // full path is passed down. Reading the flag keeps the walk pure;+ // `layer.match()` would assign `params`/`path` on the shared layer.+ const matches: Exclude<LayerMatch, false>[] =+ layer.slash === true+ ? [{ path: "" }]+ : matchers+ .map(match => match(path))+ .filter(+ (match): match is Exclude<LayerMatch, false> => match !== false,+ );++ if (matches.length === 0) {+ continue;+ }++ if (layer.route) {+ const methods = layer.route.methods ?? {};+ for (const [method, enabled] of Object.entries(methods)) {+ if (enabled && method !== "_all") {+ found.add(method.toUpperCase());+ }+ }+ continue;+ }++ // A mounted sub-router. Strip the prefix it matched and recurse.+ if (!hasStack(layer.handle)) {+ continue;+ }+ for (const matched of matches) {+ const rest = path.slice(matched.path.length);+ walk(layer.handle.stack, rest === "" ? "/" : rest, found);+ }+ }+}++/**+ * Methods registered for `path` anywhere in the app, ignoring the method of the+ * request that missed. Returns an empty set when the path itself is unknown.+ */+function allowedMethodsForPath(app: unknown, path: string): Set<string> {+ const found = new Set<string>();+ try {+ const router = (app as { router?: StackHolder }).router;+ if (router && Array.isArray(router.stack)) {+ walk(router.stack, path, found);+ }+ } catch {+ // Router internals are not part of Express's public API. If they ever move,+ // fall back to a plain 404 rather than failing the request.+ return new Set<string>();+ }+ if (found.has("GET")) {+ found.add("HEAD");+ }+ return found;+}++/**+ * Terminal handler for requests that matched no route. Replaces Express's+ * default HTML "Cannot GET /v2/scrape" page with JSON in the same envelope the+ * rest of the API uses, and answers 405 with an Allow header when the path+ * exists under a different method.+ *+ * Must be registered after every router and before the error middleware.+ */+export function notFoundHandler(+ req: Request,+ res: Response,+ _next: NextFunction,+): void {+ // `res.json` below is observed by the versioned timing middleware, which+ // otherwise labels the sample with the raw request path — one time series per+ // scanned URL. Collapse every unmatched request onto one label.+ markRouteUnmatched(req);++ // finalhandler's HTML page carried this; res.json does not set it.+ res.setHeader("X-Content-Type-Options", "nosniff");++ const allowed = [...allowedMethodsForPath(req.app, req.path)]+ .filter(method => method !== req.method)+ .sort();++ if (allowed.length > 0) {+ res.setHeader("Allow", allowed.join(", "));+ res.status(405).json({+ success: false,+ code: "METHOD_NOT_ALLOWED",+ error: `${req.method} ${req.path} is not supported. Use ${allowed.join(", ")} instead.`,+ allowed_methods: allowed,+ documentation_url: NOT_FOUND_DOCUMENTATION_URL,+ });+ return;+ }++ res.status(404).json({+ success: false,+ code: "NOT_FOUND",+ error: `${req.method} ${req.path} is not a Firecrawl API endpoint.`,+ documentation_url: NOT_FOUND_DOCUMENTATION_URL,+ });+}