Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): answer unmatched and wrong-method paths with JSON (#4552)

Himadri Mishraf31884ae57f9
diff --git a/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts b/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts
new file mode 100644
index 000000000..1217c8ace
--- /dev/null
+++ b/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts
@@ -0,0 +1,144 @@
+import express from "express";
+import request from "supertest";
+
+vi.mock("../../services/autumn/autumn.service", () => ({
+ autumnService: { checkCredits: vi.fn() },
+ CREDITS_FEATURE_ID: "CREDITS",
+}));
+vi.mock("../../services/autumn/usage", () => ({
+ getTeamBalance: vi.fn(),
+}));
+vi.mock("../../controllers/auth", () => ({
+ authenticateUser: vi.fn(),
+}));
+vi.mock("../../services/idempotency/create", () => ({
+ createIdempotencyKey: vi.fn(),
+}));
+vi.mock("../../services/idempotency/validate", () => ({
+ validateIdempotencyKey: vi.fn(),
+}));
+vi.mock("geoip-country", () => ({ lookup: vi.fn(() => null) }));
+
+import { notFoundHandler } from "../../lib/not-found";
+import { httpRequestDurationSeconds } from "../../lib/http-metrics";
+import { requestTimingMiddleware } from "../../routes/shared";
+
+// The real timing middleware patches res.json, so every JSON answer produced
+// after it runs -- including the terminal 404/405 -- is observed by the
+// histogram. Unmatched paths must not mint a new time series per URL.
+function appUnderTest() {
+ const app = express();
+
+ const v2 = express.Router();
+ v2.use(requestTimingMiddleware("v2"));
+ v2.post("/scrape", (_req, res) => res.status(200).json({ success: true }));
+ // Mirrors the real /v2/crawl/:jobId table (src/routes/v2.ts): only GET and
+ // DELETE are registered, never PATCH.
+ v2.get("/crawl/:jobId", (_req, res) => res.status(200).json({ success: true }));
+ v2.delete("/crawl/:jobId", (_req, res) => res.status(200).json({ success: true }));
+
+ app.use("/v2", v2);
+ app.use(notFoundHandler);
+ return app;
+}
+
+async function recordedRoutes(): Promise<string[]> {
+ const metric = await httpRequestDurationSeconds.get();
+ return metric.values
+ .filter(value => value.metricName === "http_request_duration_seconds_count")
+ .map(value => String(value.labels.route));
+}
+
+describe("not-found handler metric labels", () => {
+ beforeEach(() => {
+ httpRequestDurationSeconds.reset();
+ });
+
+ it("labels unknown paths with a single constant instead of the raw path", async () => {
+ const app = appUnderTest();
+
+ await request(app).get("/v2/wp-login.php");
+ await request(app).get("/v2/.env");
+ await request(app).get("/v2/some/deep/scanner/path");
+
+ const routes = await recordedRoutes();
+
+ expect(routes.length).toBeGreaterThan(0);
+ expect(new Set(routes)).toEqual(new Set(["unmatched"]));
+ });
+
+ it("labels wrong-method hits on a static path with the same constant", async () => {
+ const app = appUnderTest();
+
+ await request(app).get("/v2/scrape");
+ await request(app).delete("/v2/scrape");
+
+ const routes = await recordedRoutes();
+
+ expect(routes.length).toBeGreaterThan(0);
+ expect(new Set(routes)).toEqual(new Set(["unmatched"]));
+ });
+
+ it("labels a wrong-method hit on a genuinely parameterised path with the same constant", async () => {
+ const app = appUnderTest();
+
+ // PATCH is not registered for /v2/crawl/:jobId -- only GET and DELETE are.
+ const res = await request(app).patch("/v2/crawl/abc-123");
+
+ expect(res.status).toBe(405);
+ expect(res.headers["allow"]).toBe("DELETE, GET, HEAD");
+ expect(res.body).toEqual({
+ success: false,
+ code: "METHOD_NOT_ALLOWED",
+ error:
+ "PATCH /v2/crawl/abc-123 is not supported. Use DELETE, GET, HEAD instead.",
+ allowed_methods: ["DELETE", "GET", "HEAD"],
+ documentation_url:
+ "https://docs.firecrawl.dev/api-reference/introduction",
+ });
+
+ const routes = await recordedRoutes();
+
+ expect(routes.length).toBeGreaterThan(0);
+ expect(new Set(routes)).toEqual(new Set(["unmatched"]));
+ });
+
+ it("leaves the label of a matched route untouched", async () => {
+ const app = appUnderTest();
+
+ await request(app).post("/v2/scrape");
+
+ expect(await recordedRoutes()).toEqual(["/scrape"]);
+ });
+});
+
+describe("terminal not-found handler security headers", () => {
+ it("sets nosniff on the JSON 404", async () => {
+ const res = await request(appUnderTest()).get("/v2/nonexistent-xyz");
+
+ expect(res.status).toBe(404);
+ expect(res.headers["x-content-type-options"]).toBe("nosniff");
+ expect(res.body).toEqual({
+ success: false,
+ code: "NOT_FOUND",
+ error: "GET /v2/nonexistent-xyz is not a Firecrawl API endpoint.",
+ documentation_url:
+ "https://docs.firecrawl.dev/api-reference/introduction",
+ });
+ });
+
+ it("sets nosniff on the JSON 405", async () => {
+ const res = await request(appUnderTest()).get("/v2/scrape");
+
+ expect(res.status).toBe(405);
+ expect(res.headers["x-content-type-options"]).toBe("nosniff");
+ expect(res.body).toEqual({
+ success: false,
+ code: "METHOD_NOT_ALLOWED",
+ error: "GET /v2/scrape is not supported. Use POST instead.",
+ allowed_methods: ["POST"],
+ documentation_url:
+ "https://docs.firecrawl.dev/api-reference/introduction",
+ });
+ });
+});
diff --git a/apps/api/src/__tests__/routes/not-found.routes.test.ts b/apps/api/src/__tests__/routes/not-found.routes.test.ts
new file mode 100644
index 000000000..9811a18fb
--- /dev/null
+++ b/apps/api/src/__tests__/routes/not-found.routes.test.ts
@@ -0,0 +1,181 @@
+import express from "express";
+import request from "supertest";
+import { notFoundHandler } from "../../lib/not-found";
+
+// A stand-in for the real app's mount structure: a versioned router mounted
+// under a prefix, a sub-router mounted inside it, and a root descriptor. The
+// real routers pull in the whole service graph, so the shape is replayed here
+// rather than imported.
+function appUnderTest() {
+ const app = express();
+
+ app.get("/", (_req, res) => {
+ res.json({
+ message: "Firecrawl API",
+ documentation_url: "https://docs.firecrawl.dev",
+ });
+ });
+
+ const nested = express.Router();
+ nested.post("/", (_req, res) => res.status(200).json({ success: true }));
+ nested.delete("/", (_req, res) => res.status(200).json({ success: true }));
+
+ const v2 = express.Router();
+ v2.post("/scrape", (_req, res) => res.status(200).json({ success: true }));
+ v2.post("/search", (_req, res) => res.status(200).json({ success: true }));
+ v2.get("/crawl/:id", (_req, res) => res.status(200).json({ success: true }));
+ v2.delete("/crawl/:id", (_req, res) =>
+ res.status(200).json({ success: true }),
+ );
+ v2.post("/scrape/:jobId/interact", (_req, res) =>
+ res.status(200).json({ success: true }),
+ );
+ v2.delete("/scrape/:jobId/interact", (_req, res) =>
+ res.status(200).json({ success: true }),
+ );
+ v2.use("/developer", nested);
+
+ // Mounted with no path, the way `v0Router` and `adminRouter` are in
+ // src/index.ts. Express flags such a layer as matching everything and never
+ // consults its matchers, so the walk has to handle it explicitly.
+ const rootMounted = express.Router();
+ rootMounted.post("/v0/scrape", (_req, res) =>
+ res.status(200).json({ success: true }),
+ );
+ rootMounted.get("/v0/health/liveness", (_req, res) =>
+ res.status(200).json({ success: true }),
+ );
+
+ const adminMounted = express.Router();
+ adminMounted.post("/admin/:key/queues", (_req, res) =>
+ res.status(200).json({ success: true }),
+ );
+
+ app.use("/v2", v2);
+ app.use(rootMounted);
+ app.use(adminMounted);
+ app.use(notFoundHandler);
+ return app;
+}
+
+describe("terminal not-found handler", () => {
+ it("answers an unknown path with JSON, not Express's HTML page", async () => {
+ const res = await request(appUnderTest()).get("/v2/nonexistent-xyz");
+
+ expect(res.status).toBe(404);
+ expect(res.headers["content-type"]).toMatch(/application\/json/);
+ expect(res.text).not.toContain("Cannot GET");
+ expect(res.body).toEqual({
+ success: false,
+ code: "NOT_FOUND",
+ error: "GET /v2/nonexistent-xyz is not a Firecrawl API endpoint.",
+ documentation_url:
+ "https://docs.firecrawl.dev/api-reference/introduction",
+ });
+ });
+
+ it.each(["/v2/scrape", "/v2/search"])(
+ "answers GET %s with 405 and an Allow header",
+ async path => {
+ const res = await request(appUnderTest()).get(path);
+
+ expect(res.status).toBe(405);
+ expect(res.headers["content-type"]).toMatch(/application\/json/);
+ expect(res.headers["allow"]).toBe("POST");
+ expect(res.text).not.toContain("Cannot GET");
+ expect(res.body.code).toBe("METHOD_NOT_ALLOWED");
+ expect(res.body.allowed_methods).toEqual(["POST"]);
+ expect(res.body.documentation_url).toBe(
+ "https://docs.firecrawl.dev/api-reference/introduction",
+ );
+ },
+ );
+
+ it("lists every other method for a path that has several", async () => {
+ const res = await request(appUnderTest()).post("/v2/crawl/abc");
+
+ expect(res.status).toBe(405);
+ expect(res.headers["allow"]).toBe("DELETE, GET, HEAD");
+ expect(res.body.allowed_methods).toEqual(["DELETE", "GET", "HEAD"]);
+ });
+
+ it("answers GET on the interact path with 405, not Cannot GET", async () => {
+ const res = await request(appUnderTest()).get("/v2/scrape/abc/interact");
+
+ expect(res.status).toBe(405);
+ expect(res.text).not.toContain("Cannot GET");
+ expect(res.headers["allow"]).toBe("DELETE, POST");
+ });
+
+ it("resolves paths inside a mounted sub-router", async () => {
+ const res = await request(appUnderTest()).get("/v2/developer");
+
+ expect(res.status).toBe(405);
+ expect(res.headers["allow"]).toBe("DELETE, POST");
+ });
+
+ it("answers GET on a POST-only route in a root-mounted router with 405", async () => {
+ const res = await request(appUnderTest()).get("/v0/scrape");
+
+ expect(res.status).toBe(405);
+ expect(res.headers["allow"]).toBe("POST");
+ expect(res.text).not.toContain("Cannot GET");
+ expect(res.body.code).toBe("METHOD_NOT_ALLOWED");
+ expect(res.body.allowed_methods).toEqual(["POST"]);
+ });
+
+ it("answers POST on a GET-only route in a root-mounted router with 405", async () => {
+ const res = await request(appUnderTest()).post("/v0/health/liveness");
+
+ expect(res.status).toBe(405);
+ expect(res.headers["allow"]).toBe("GET, HEAD");
+ expect(res.body.allowed_methods).toEqual(["GET", "HEAD"]);
+ });
+
+ it("resolves admin paths in a second root-mounted router", async () => {
+ const res = await request(appUnderTest()).get("/admin/secret/queues");
+
+ expect(res.status).toBe(405);
+ expect(res.headers["allow"]).toBe("POST");
+ });
+
+ it("still answers an unknown path under a root mount with a JSON 404", async () => {
+ const res = await request(appUnderTest()).get("/v0/nonexistent-xyz");
+
+ expect(res.status).toBe(404);
+ expect(res.headers["content-type"]).toMatch(/application\/json/);
+ expect(res.headers["allow"]).toBeUndefined();
+ expect(res.body.code).toBe("NOT_FOUND");
+ });
+
+ it("leaves existing routes alone", async () => {
+ const app = appUnderTest();
+
+ const scrape = await request(app).post("/v2/scrape");
+ expect(scrape.status).toBe(200);
+ expect(scrape.body).toEqual({ success: true });
+
+ const status = await request(app).get("/v2/crawl/abc");
+ expect(status.status).toBe(200);
+
+ const interact = await request(app).post("/v2/scrape/abc/interact");
+ expect(interact.status).toBe(200);
+
+ const developer = await request(app).post("/v2/developer");
+ expect(developer.status).toBe(200);
+
+ const root = await request(app).get("/");
+ expect(root.status).toBe(200);
+ expect(root.body.message).toBe("Firecrawl API");
+
+ const v0Scrape = await request(app).post("/v0/scrape");
+ expect(v0Scrape.status).toBe(200);
+ expect(v0Scrape.body).toEqual({ success: true });
+
+ const liveness = await request(app).get("/v0/health/liveness");
+ expect(liveness.status).toBe(200);
+
+ const admin = await request(app).post("/admin/secret/queues");
+ expect(admin.status).toBe(200);
+ });
+});
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 4c8af690c..8beb321a2 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -45,6 +45,7 @@ import { shutdownWebhookQueue } from "./services/webhook";
import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";
import { isKeylessConfigured } from "./lib/keyless";
import { shutdownPubSubLogging } from "./services/logging/log_job";
+import { notFoundHandler } from "./lib/not-found";
const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");
@@ -212,6 +213,10 @@ app.get("/is-production", (req, res) => {
res.send({ isProduction: global.isProduction });
});
+// Terminal handler for unmatched paths. Must stay after every route and before
+// the error middleware below, or Express falls back to its default HTML page.
+app.use(notFoundHandler);
+
app.use(
(
err: unknown,
diff --git a/apps/api/src/lib/http-metrics.ts b/apps/api/src/lib/http-metrics.ts
index 6e8ee366a..e3808b5cf 100644
--- a/apps/api/src/lib/http-metrics.ts
+++ b/apps/api/src/lib/http-metrics.ts
@@ -11,9 +11,27 @@ export const httpRequestDurationSeconds = new Histogram({
buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],
});
+/**
+ * Single `route` label for every request that matched no route. Without it the
+ * fallback below would mint a new time series per scanned URL, since unmatched
+ * requests reach the terminal handler and answer with `res.json`, which the
+ * timing middleware observes.
+ */
+const UNMATCHED_ROUTE = "unmatched";
+
+const UNMATCHED_FLAG = Symbol.for("firecrawl.httpMetrics.unmatchedRoute");
+
+/** Called by the terminal not-found handler before it answers. */
+export function markRouteUnmatched(req: Request): void {
+ (req as unknown as Record<symbol, boolean>)[UNMATCHED_FLAG] = true;
+}
+
export function getRoutePattern(req: Request): string {
if (req.route?.path) {
return req.route.path;
}
+ if ((req as unknown as Record<symbol, boolean>)[UNMATCHED_FLAG] === true) {
+ return UNMATCHED_ROUTE;
+ }
return req.path.replace(UUID_REGEX, ":id");
}
diff --git a/apps/api/src/lib/not-found.ts b/apps/api/src/lib/not-found.ts
new file mode 100644
index 000000000..4e5a00048
--- /dev/null
+++ b/apps/api/src/lib/not-found.ts
@@ -0,0 +1,138 @@
+import type { NextFunction, Request, Response } from "express";
+import { markRouteUnmatched } from "./http-metrics";
+
+const NOT_FOUND_DOCUMENTATION_URL =
+ "https://docs.firecrawl.dev/api-reference/introduction";
+
+// Express 5 router internals. Layers expose pure matcher functions, so this
+// walk never mutates routing state.
+type LayerMatch = { path: string } | false;
+type Matcher = (path: string) => LayerMatch;
+type Layer = {
+ matchers?: Matcher[];
+ // Set by Express for a router mounted with no path (`app.use(router)`).
+ // Express short-circuits on this flag and never consults `matchers`, whose
+ // patterns only match "/" for such a layer.
+ slash?: boolean;
+ route?: { methods?: Record<string, boolean> };
+ handle?: unknown;
+};
+type StackHolder = { stack?: Layer[] };
+
+function hasStack(value: unknown): value is { stack: Layer[] } {
+ if (typeof value !== "function" && typeof value !== "object") {
+ return false;
+ }
+ return value !== null && Array.isArray((value as StackHolder).stack);
+}
+
+function walk(stack: Layer[], path: string, found: Set<string>): void {
+ for (const layer of stack) {
+ const matchers = layer.matchers;
+ if (!Array.isArray(matchers)) {
+ continue;
+ }
+
+ // A root-mounted layer matches every path and consumes no prefix, so the
+ // full path is passed down. Reading the flag keeps the walk pure;
+ // `layer.match()` would assign `params`/`path` on the shared layer.
+ const matches: Exclude<LayerMatch, false>[] =
+ layer.slash === true
+ ? [{ path: "" }]
+ : matchers
+ .map(match => match(path))
+ .filter(
+ (match): match is Exclude<LayerMatch, false> => match !== false,
+ );
+
+ if (matches.length === 0) {
+ continue;
+ }
+
+ if (layer.route) {
+ const methods = layer.route.methods ?? {};
+ for (const [method, enabled] of Object.entries(methods)) {
+ if (enabled && method !== "_all") {
+ found.add(method.toUpperCase());
+ }
+ }
+ continue;
+ }
+
+ // A mounted sub-router. Strip the prefix it matched and recurse.
+ if (!hasStack(layer.handle)) {
+ continue;
+ }
+ for (const matched of matches) {
+ const rest = path.slice(matched.path.length);
+ walk(layer.handle.stack, rest === "" ? "/" : rest, found);
+ }
+ }
+}
+
+/**
+ * Methods registered for `path` anywhere in the app, ignoring the method of the
+ * request that missed. Returns an empty set when the path itself is unknown.
+ */
+function allowedMethodsForPath(app: unknown, path: string): Set<string> {
+ const found = new Set<string>();
+ try {
+ const router = (app as { router?: StackHolder }).router;
+ if (router && Array.isArray(router.stack)) {
+ walk(router.stack, path, found);
+ }
+ } catch {
+ // Router internals are not part of Express's public API. If they ever move,
+ // fall back to a plain 404 rather than failing the request.
+ return new Set<string>();
+ }
+ if (found.has("GET")) {
+ found.add("HEAD");
+ }
+ return found;
+}
+
+/**
+ * Terminal handler for requests that matched no route. Replaces Express's
+ * default HTML "Cannot GET /v2/scrape" page with JSON in the same envelope the
+ * rest of the API uses, and answers 405 with an Allow header when the path
+ * exists under a different method.
+ *
+ * Must be registered after every router and before the error middleware.
+ */
+export function notFoundHandler(
+ req: Request,
+ res: Response,
+ _next: NextFunction,
+): void {
+ // `res.json` below is observed by the versioned timing middleware, which
+ // otherwise labels the sample with the raw request path — one time series per
+ // scanned URL. Collapse every unmatched request onto one label.
+ markRouteUnmatched(req);
+
+ // finalhandler's HTML page carried this; res.json does not set it.
+ res.setHeader("X-Content-Type-Options", "nosniff");
+
+ const allowed = [...allowedMethodsForPath(req.app, req.path)]
+ .filter(method => method !== req.method)
+ .sort();
+
+ if (allowed.length > 0) {
+ res.setHeader("Allow", allowed.join(", "));
+ res.status(405).json({
+ success: false,
+ code: "METHOD_NOT_ALLOWED",
+ error: `${req.method} ${req.path} is not supported. Use ${allowed.join(", ")} instead.`,
+ allowed_methods: allowed,
+ documentation_url: NOT_FOUND_DOCUMENTATION_URL,
+ });
+ return;
+ }
+
+ res.status(404).json({
+ success: false,
+ code: "NOT_FOUND",
+ error: `${req.method} ${req.path} is not a Firecrawl API endpoint.`,
+ documentation_url: NOT_FOUND_DOCUMENTATION_URL,
+ });
+}