Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add integration field to jobs and update related controllers and types
AdemÃlson Tonato57a0aed48421
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex e70c79d9d..92b452108 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -204,6 +204,7 @@ export async function crawlController(req: Request, res: Response) {internalOptions,team_id,origin: req.body.origin ?? defaultOrigin,+ integration: req.body.integration,crawl_id: id,sitemapped: true,},@@ -245,6 +246,7 @@ export async function crawlController(req: Request, res: Response) {internalOptions,team_id,origin: req.body.origin ?? defaultOrigin,+ integration: req.body.integration,crawl_id: id,},{diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 559d222cc..9d1ce86dd 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -81,6 +81,7 @@ export async function scrapeHelper(scrapeOptions,internalOptions,origin: req.body.origin ?? defaultOrigin,+ integration: req.body.integration,is_scrape: true,startTime: Date.now(),},diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex e4be9bd75..3bbb45ce5 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -222,7 +222,8 @@ export async function searchController(req: Request, res: Response) {url: req.body.query,scrapeOptions: fromLegacyScrapeOptions(req.body.pageOptions, undefined, 60000, team_id),crawlerOptions: crawlerOptions,- origin: origin,+ origin,+ integration: req.body.integration,});return res.status(result.returnCode).json(result);} catch (error) {diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex d14f8cd7f..c08ea93b6 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -85,6 +85,7 @@ export async function batchScrapeController(appendToId: req.body.appendToId,account: req.account,});+ logger.debug("[batch-scrape - batchScrapeController] Integration: " + req.body.integration);if (!req.body.appendToId) {await logCrawl(id, req.auth.team_id);@@ -134,6 +135,7 @@ export async function batchScrapeController(crawlerOptions: null,scrapeOptions,origin: "api",+ integration: req.body.integration,crawl_id: id,sitemapped: true,v1: true,diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex fe9d6afd9..fb178bbfe 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -31,6 +31,7 @@ export async function crawlController(originalRequest: preNormalizedBody,account: req.account,});+ logger.debug("[crawl - crawlController] Integration: " + req.body.integration);await logCrawl(id, req.auth.team_id);@@ -114,6 +115,7 @@ export async function crawlController(scrapeOptions: sc.scrapeOptions,internalOptions: sc.internalOptions,origin: req.body.origin,+ integration: req.body.integration,crawl_id: id,webhook: req.body.webhook,v1: true,diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 4ffaddfe6..19cdc9fa5 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -83,6 +83,7 @@ export async function extractController(subId: req.acuc?.sub_id,extractId,});+ _logger.debug("[extract - extractController] Integration: " + req.body.integration);const jobData = {request: req.body,diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex a7a47d05e..8697b7ddb 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -331,6 +331,7 @@ export async function mapController(originalRequest,teamId: req.auth.team_id,});+ logger.debug("[map - mapController] Integration: " + req.body.integration);let result: Awaited<ReturnType<typeof getMapResults>>;const abort = new AbortController();@@ -390,6 +391,7 @@ export async function mapController(crawlerOptions: {},scrapeOptions: {},origin: req.body.origin ?? "api",+ integration: req.body.integration,num_tokens: 0,});diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex a97d0e63d..9967043c5 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -29,6 +29,7 @@ export async function scrapeController(teamId: req.auth.team_id,account: req.account,});+ logger.debug("[scrape - scrapeController] Integration: " + req.body.integration);req.body = scrapeRequestSchema.parse(req.body);@@ -57,7 +58,8 @@ export async function scrapeController(useCache: req.body.__experimental_cache ? true : false,bypassBilling: isDirectToBullMQ,},- origin: req.body.origin,+ origin,+ integration: req.body.integration,startTime,},{},diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex ceacce4cc..f45e5eebe 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -181,6 +181,7 @@ export async function searchController(query: req.body.query,origin: req.body.origin,});+ logger.debug("[search - searchController] Integration: " + req.body.integration);let limit = req.body.limit;@@ -307,6 +308,7 @@ export async function searchController(url: req.body.query,scrapeOptions: req.body.scrapeOptions,origin: req.body.origin,+ integration: req.body.integration,cost_tracking: costTracking,},false,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 88433b4c4..06768ffb0 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -10,6 +10,22 @@ import {} from "../../lib/entities";import { InternalOptions } from "../../scraper/scrapeURL";+export enum IntegrationEnum {+ DIFY = "dify",+ ZAPIER = "zapier",+ PIPEDREAM = "pipedream",+ RAYCAST = "raycast",+ LANGCHAIN = "langchain",+ CREWAI = "crewai",+ LLAMAINDEX = "llamaindex",+ N8N = "n8n",+ CAMELAI = "camelai",+ MAKE = "make",+ FLOWISE = "flowise",+ METAGPT = "metagpt",+ UNKNOWN = "unknown",+}+export type Format =| "markdown"| "html"@@ -469,6 +485,7 @@ export const extractV1Options = zenableWebSearch: z.boolean().default(false),scrapeOptions: baseScrapeOptions.default({ onlyMainContent: false }).optional(),origin: z.string().optional().default("api"),+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),urlTrace: z.boolean().default(false),timeout: z.number().int().positive().finite().safe().default(60000),__experimental_streamSteps: z.boolean().default(false),@@ -527,6 +544,7 @@ export const scrapeRequestSchema = baseScrapeOptionsextract: extractOptionsWithAgent.optional(),jsonOptions: extractOptionsWithAgent.optional(),origin: z.string().optional().default("api"),+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),timeout: z.number().int().positive().finite().safe().default(30000),}).strict(strictMessage)@@ -561,6 +579,7 @@ export const batchScrapeRequestSchema = baseScrapeOptions.extend({urls: url.array(),origin: z.string().optional().default("api"),+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),webhook: webhookSchema.optional(),appendToId: z.string().uuid().optional(),ignoreInvalidURLs: z.boolean().default(false),@@ -574,6 +593,7 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions.extend({urls: z.string().array(),origin: z.string().optional().default("api"),+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),webhook: webhookSchema.optional(),appendToId: z.string().uuid().optional(),ignoreInvalidURLs: z.boolean().default(false),@@ -621,6 +641,7 @@ export const crawlRequestSchema = crawlerOptions.extend({url,origin: z.string().optional().default("api"),+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),scrapeOptions: baseScrapeOptions.default({}),webhook: webhookSchema.optional(),limit: z.number().default(10000),@@ -652,6 +673,7 @@ export const mapRequestSchema = crawlerOptions.extend({url,origin: z.string().optional().default("api"),+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),includeSubdomains: z.boolean().default(true),search: z.string().optional(),ignoreSitemap: z.boolean().default(false),@@ -1172,6 +1194,7 @@ export const searchRequestSchema = zcountry: z.string().optional().default("us"),location: z.string().optional(),origin: z.string().optional().default("api"),+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),timeout: z.number().int().positive().finite().safe().default(60000),ignoreInvalidURLs: z.boolean().optional().default(false),__searchPreviewToken: z.string().optional(),diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 1bd73dd81..6b6e23841 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -185,6 +185,7 @@ export async function performExtraction(url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: 0,tokens_billed,sources,@@ -680,6 +681,7 @@ export async function performExtraction(url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: 0,tokens_billed,sources,@@ -787,6 +789,7 @@ export async function performExtraction(url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: 0,tokens_billed,sources,@@ -827,6 +830,7 @@ export async function performExtraction(url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: 0,tokens_billed,sources,@@ -1011,6 +1015,7 @@ export async function performExtraction(url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: totalTokensUsed,tokens_billed: tokensToBill,sources,@@ -1079,6 +1084,7 @@ export async function performExtraction(url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: 0,tokens_billed,sources,diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 1ddf17f38..3f4e041ee 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -118,6 +118,7 @@ import { getACUCTeam } from "../../../controllers/auth";url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: 0,tokens_billed: 0,sources,@@ -219,6 +220,7 @@ import { getACUCTeam } from "../../../controllers/auth";url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: 0,tokens_billed: 0,sources,@@ -568,6 +570,7 @@ import { getACUCTeam } from "../../../controllers/auth";url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: 0,tokens_billed: 0,sources,@@ -663,6 +666,7 @@ import { getACUCTeam } from "../../../controllers/auth";url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: 0,tokens_billed: 0,sources,@@ -691,6 +695,7 @@ import { getACUCTeam } from "../../../controllers/auth";url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: 0,tokens_billed: 0,sources,@@ -852,6 +857,7 @@ import { getACUCTeam } from "../../../controllers/auth";url: request.urls?.join(", ") || "",scrapeOptions: request,origin: request.origin ?? "api",+ integration: request.integration,num_tokens: totalTokensUsed,tokens_billed: tokensToBill,sources,diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.tsindex 3e780457b..92047f1ea 100644--- a/apps/api/src/lib/gcs-jobs.ts+++ b/apps/api/src/lib/gcs-jobs.ts@@ -48,6 +48,7 @@ export async function saveJobToGCS(job: FirecrawlJob): Promise<void> {crawler_options: JSON.stringify(job.crawlerOptions),page_options: JSON.stringify(job.scrapeOptions),origin: job.origin,+ integration: job.integration ?? null,num_tokens: job.num_tokens ?? null,retry: !!job.retry,crawl_id: job.crawl_id ?? null,diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 262d4015d..82e5e07b9 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -57,6 +57,7 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLocrawler_options: job.crawlerOptions,page_options: job.scrapeOptions,origin: job.origin,+ integration: job.integration ?? null,num_tokens: job.num_tokens,retry: !!job.retry,crawl_id: job.crawl_id,diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 2a1bdf665..a20e39ab3 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -321,6 +321,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {scrapeOptions: sc.scrapeOptions,crawlerOptions: sc.crawlerOptions,origin: job.data.origin,+ integration: job.data.integration,}, false, job.data.internalOptions?.bypassBilling ?? false);logger.info("Logged crawl!");@@ -371,6 +372,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {(job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),crawlerOptions: sc.crawlerOptions,origin: job.data.origin,+ integration: job.data.integration,},true,job.data.internalOptions?.bypassBilling ?? false,@@ -1454,6 +1456,7 @@ async function processJob(job: Job & { id: string }, token: string) {crawlerOptions: sc.crawlerOptions,scrapeOptions: job.data.scrapeOptions,origin: job.data.origin,+ integration: job.data.integration,crawl_id: job.data.crawl_id,cost_tracking: costTracking,pdf_num_pages: doc.metadata.numPages,@@ -1503,6 +1506,7 @@ async function processJob(job: Job & { id: string }, token: string) {url: job.data.url,scrapeOptions: job.data.scrapeOptions,origin: job.data.origin,+ integration: job.data.integration,num_tokens: 0, // TODO: fixcost_tracking: costTracking,pdf_num_pages: doc.metadata.numPages,@@ -1602,6 +1606,7 @@ async function processJob(job: Job & { id: string }, token: string) {crawlerOptions: job.data.crawlerOptions,scrapeOptions: job.data.scrapeOptions,origin: job.data.origin,+ integration: job.data.integration,crawl_id: job.data.crawl_id,cost_tracking: costTracking,},diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 2e18ba3d9..3167e7880 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -44,6 +44,7 @@ export interface WebScraperOptions {sitemapped?: boolean;webhook?: z.infer<typeof webhookSchema>;v1?: boolean;+ integration?: string;/*** Disables billing on the worker side.@@ -94,6 +95,7 @@ export interface FirecrawlJob {crawlerOptions?: any;scrapeOptions?: any;origin: string;+ integration?: string;num_tokens?: number;retry?: boolean;crawl_id?: string;