Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): write ledger tracks via HTTP endpoint
firecrawl-spring[bot]e68ac5bbc88c
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 2d2a600ff..a7cf7fbdb 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -31,6 +31,14 @@ const configSchema = z.object({FIRECRAWL_DASHBOARD_URL: z.url().default("https://www.firecrawl.dev"),SUPPORT_AGENT_URL: z.string().url().optional(),SUPPORT_AGENT_VERCEL_BYPASS_SECRET: z.string().optional(),+ FIREBRAIN_TRACKS_URL: z.preprocess(+ v => (typeof v === "string" && v.trim() === "" ? undefined : v),+ z.string().url().optional(),+ ),+ FIREBRAIN_TRACKS_API_KEY: z.preprocess(+ v => (typeof v === "string" && v.trim() === "" ? undefined : v),+ z.string().trim().optional(),+ ),RESEARCH_PROXY_URL: z.string().url().optional(),LABS_SEARCH_URL: z.string().url().optional(),LABS_SEARCH_SECRET: z.string().optional(),diff --git a/apps/api/src/db/schema/index.ts b/apps/api/src/db/schema/index.tsindex 402cd0bb1..25687c02b 100644--- a/apps/api/src/db/schema/index.ts+++ b/apps/api/src/db/schema/index.ts@@ -1,3 +1,2 @@export * from "./public";-export * from "./ledger";export * from "./index-db";diff --git a/apps/api/src/db/schema/ledger.ts b/apps/api/src/db/schema/ledger.tsdeleted file mode 100644index cc4a4da9d..000000000--- a/apps/api/src/db/schema/ledger.ts+++ /dev/null@@ -1,40 +0,0 @@-import {- pgSchema,- bigint,- boolean,- text,- timestamp,- uuid,- jsonb,-} from "drizzle-orm/pg-core";--const ledger = pgSchema("ledger");--const ts = (name: string) =>- timestamp(name, { withTimezone: true, mode: "string" });-const bigintNum = (name: string) => bigint(name, { mode: "number" });--export const provider_definitions = ledger.table("provider_definitions", {- id: bigintNum("id").notNull().generatedAlwaysAsIdentity(),- created_at: ts("created_at").notNull().defaultNow(),- updated_at: ts("updated_at").notNull().defaultNow(),- is_active: boolean("is_active").notNull().default(true),- provider_id: bigintNum("provider_id").notNull(),- slug: text("slug").notNull(),- name: text("name").notNull(),- description: text("description"),- provider_definition_type_id: bigintNum(- "provider_definition_type_id",- ).notNull(),-});--export const tracks = ledger.table("tracks", {- id: bigintNum("id").notNull().generatedAlwaysAsIdentity(),- is_active: boolean("is_active").notNull().default(true),- created_at: ts("created_at").notNull().defaultNow(),- updated_at: ts("updated_at").notNull().defaultNow(),- received_at: ts("received_at").notNull().defaultNow(),- provider_definition_id: bigintNum("provider_definition_id").notNull(),- uuid: uuid("uuid").notNull().defaultRandom(),- data: jsonb("data").notNull().default({}),-});diff --git a/apps/api/src/services/ledger/tracking.test.ts b/apps/api/src/services/ledger/tracking.test.tsnew file mode 100644index 000000000..81dcf6a08--- /dev/null+++ b/apps/api/src/services/ledger/tracking.test.ts@@ -0,0 +1,74 @@+import { vi } from "vitest";++vi.mock("../../config", () => ({+ config: {+ FIREBRAIN_TRACKS_URL: "https://tracks.example.test/store",+ FIREBRAIN_TRACKS_API_KEY: "fb_test_key",+ },+}));++vi.mock("../../lib/logger", () => ({+ logger: { warn: vi.fn(), error: vi.fn(), info: vi.fn(), debug: vi.fn() },+}));++import { trackEvent } from "./tracking";+import { config } from "../../config";++const fetchMock = vi.fn();+vi.stubGlobal("fetch", fetchMock);++beforeEach(() => {+ vi.clearAllMocks();+ (config as any).FIREBRAIN_TRACKS_URL = "https://tracks.example.test/store";+ (config as any).FIREBRAIN_TRACKS_API_KEY = "fb_test_key";+});++describe("trackEvent", () => {+ it("POSTs the firecrawl-api envelope and returns the track uuid on success", async () => {+ fetchMock.mockResolvedValue({+ ok: true,+ status: 200,+ json: async () => ({ tracks: [{ track_uuid: "uuid-1" }] }),+ text: async () => "",+ });++ const result = await trackEvent("concurrent-browser-limit-reached", {+ team_id: "team_1",+ });++ expect(result).toBe("uuid-1");+ expect(fetchMock).toHaveBeenCalledTimes(1);+ const [url, init] = fetchMock.mock.calls[0];+ expect(url).toBe("https://tracks.example.test/store");+ expect(init.headers.authorization).toBe("Bearer fb_test_key");+ expect(JSON.parse(init.body)).toEqual({+ definition: "concurrent-browser-limit-reached",+ source: "firecrawl-api",+ properties: { team_id: "team_1" },+ });+ });++ it("returns null without throwing on a non-2xx response", async () => {+ fetchMock.mockResolvedValue({+ ok: false,+ status: 500,+ text: async () => "upstream error",+ json: async () => ({}),+ });++ await expect(+ trackEvent("concurrent-browser-limit-reached", { team_id: "team_1" }),+ ).resolves.toBeNull();+ });++ it("skips the request and returns null when the endpoint is unconfigured", async () => {+ (config as any).FIREBRAIN_TRACKS_URL = undefined;++ const result = await trackEvent("concurrent-browser-limit-reached", {+ team_id: "team_1",+ });++ expect(result).toBeNull();+ expect(fetchMock).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/services/ledger/tracking.ts b/apps/api/src/services/ledger/tracking.tsindex 8aba5fd7f..01da14261 100644--- a/apps/api/src/services/ledger/tracking.ts+++ b/apps/api/src/services/ledger/tracking.ts@@ -1,82 +1,69 @@import { EventDataMap, EventDefinitionSlug } from "./data-schemas";-import { eq } from "drizzle-orm";-import { db } from "../../db/connection";-import * as schema from "../../db/schema";-import { getValue, setValue } from "../redis";+import { config } from "../../config";import { logger } from "../../lib/logger";+const TRACK_SOURCE = "firecrawl-api";+const TRACK_TIMEOUT_MS = 5000;++type TrackStoreResponse = {+ tracks?: Array<{ track_uuid: string | null }>;+};+/**- * Track an event in the ledger system- * @param definitionSlug The provider definition slug- * @param data Additional data to store with the track- * @returns The tracked event ID or null if tracking failed+ * Emit a track by POSTing it to the configured tracks endpoint. Never throws:+ * on any failure it logs and returns null, so tracking cannot break the calling+ * flow.+ * @param definitionSlug The event definition slug+ * @param data Event properties sent with the track+ * @returns The stored track UUID, or null if the write did not succeed*/export async function trackEvent<T extends EventDefinitionSlug>(definitionSlug: T,data: EventDataMap[T],): Promise<string | null> {- try {- // Get the provider definition ID from cache or database- const cacheKey = `provider_definition_${definitionSlug}_`;- let providerDefinition: any = null;- let definitionError: any = null;-- // Try to get from Redis cache first- const cachedData = await getValue(cacheKey);- if (cachedData) {- providerDefinition = JSON.parse(cachedData);- } else {- // If not in cache, fetch from database- try {- [providerDefinition] = await db- .select({ id: schema.provider_definitions.id })- .from(schema.provider_definitions)- .where(eq(schema.provider_definitions.slug, definitionSlug))- .limit(1);- } catch (error) {- definitionError = error;- }-- // Cache the result for 24 hours (1440 minutes)- if (!definitionError && providerDefinition) {- await setValue(- cacheKey,- JSON.stringify(providerDefinition),- 600 * 60 * 24,- );- }- }+ const url = config.FIREBRAIN_TRACKS_URL;+ const apiKey = config.FIREBRAIN_TRACKS_API_KEY?.trim();+ if (!url || !apiKey) {+ logger.warn(+ "Skipping track: FIREBRAIN_TRACKS_URL or FIREBRAIN_TRACKS_API_KEY is unset",+ );+ return null;+ }- if (definitionError || !providerDefinition) {- logger.error("Error finding provider definition:", definitionError);- return null;- }+ try {+ const response = await fetch(url, {+ method: "POST",+ headers: {+ "content-type": "application/json",+ authorization: `Bearer ${apiKey}`,+ },+ body: JSON.stringify({+ definition: definitionSlug,+ source: TRACK_SOURCE,+ properties: data,+ }),+ signal: AbortSignal.timeout(TRACK_TIMEOUT_MS),+ });- // Create the track- let track: { uuid: string } | undefined;- try {- [track] = await db- .insert(schema.tracks)- .values({- created_at: new Date().toISOString(),- provider_definition_id: providerDefinition.id,- data: data,- })- .returning({ uuid: schema.tracks.uuid });- } catch (trackError) {- logger.error("Error creating track:", trackError);+ if (!response.ok) {+ const detail = await response.text().catch(() => "");+ logger.error("Error storing track", {+ module: "ledger/tracking",+ definitionSlug,+ status: response.status,+ detail: detail.slice(0, 300),+ });return null;}- if (!track) {- return null;- }- return track.uuid;+ const result = (await response.json()) as TrackStoreResponse;+ return result.tracks?.[0]?.track_uuid ?? null;} catch (error) {- logger.error("Error tracking event:", error);+ logger.error("Error tracking event", {+ module: "ledger/tracking",+ definitionSlug,+ error,+ });return null;}}--// data schemas?-// everything that sends an email, move to tracks