Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

refactor: remove unnecessary logs and set integration as default to null

Ademílson Tonato4c49bb9fc684
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index c08ea93b6..5016e2ab7 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -85,7 +85,6 @@ export async function batchScrapeController(
appendToId: req.body.appendToId,
account: req.account,
});
- logger.debug("[batch-scrape - batchScrapeController] Integration: " + req.body.integration);
if (!req.body.appendToId) {
await logCrawl(id, req.auth.team_id);
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index fb178bbfe..81234b2d6 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -31,7 +31,6 @@ export async function crawlController(
originalRequest: preNormalizedBody,
account: req.account,
});
- logger.debug("[crawl - crawlController] Integration: " + req.body.integration);
await logCrawl(id, req.auth.team_id);
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 19cdc9fa5..4ffaddfe6 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -83,7 +83,6 @@ export async function extractController(
subId: req.acuc?.sub_id,
extractId,
});
- _logger.debug("[extract - extractController] Integration: " + req.body.integration);
const jobData = {
request: req.body,
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 8697b7ddb..332fade6d 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -331,7 +331,6 @@ export async function mapController(
originalRequest,
teamId: req.auth.team_id,
});
- logger.debug("[map - mapController] Integration: " + req.body.integration);
let result: Awaited<ReturnType<typeof getMapResults>>;
const abort = new AbortController();
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 9967043c5..fb68f7ffb 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -29,7 +29,6 @@ export async function scrapeController(
teamId: req.auth.team_id,
account: req.account,
});
- logger.debug("[scrape - scrapeController] Integration: " + req.body.integration);
req.body = scrapeRequestSchema.parse(req.body);
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index f45e5eebe..c94480db6 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -181,7 +181,6 @@ export async function searchController(
query: req.body.query,
origin: req.body.origin,
});
- logger.debug("[search - searchController] Integration: " + req.body.integration);
let limit = req.body.limit;
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 06768ffb0..d8a5fefc5 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -23,7 +23,6 @@ export enum IntegrationEnum {
MAKE = "make",
FLOWISE = "flowise",
METAGPT = "metagpt",
- UNKNOWN = "unknown",
}
export type Format =
@@ -485,7 +484,7 @@ export const extractV1Options = z
enableWebSearch: z.boolean().default(false),
scrapeOptions: baseScrapeOptions.default({ onlyMainContent: false }).optional(),
origin: z.string().optional().default("api"),
- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
urlTrace: z.boolean().default(false),
timeout: z.number().int().positive().finite().safe().default(60000),
__experimental_streamSteps: z.boolean().default(false),
@@ -544,7 +543,7 @@ export const scrapeRequestSchema = baseScrapeOptions
extract: extractOptionsWithAgent.optional(),
jsonOptions: extractOptionsWithAgent.optional(),
origin: z.string().optional().default("api"),
- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
timeout: z.number().int().positive().finite().safe().default(30000),
})
.strict(strictMessage)
@@ -579,7 +578,7 @@ export const batchScrapeRequestSchema = baseScrapeOptions
.extend({
urls: url.array(),
origin: z.string().optional().default("api"),
- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
webhook: webhookSchema.optional(),
appendToId: z.string().uuid().optional(),
ignoreInvalidURLs: z.boolean().default(false),
@@ -593,7 +592,7 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions
.extend({
urls: z.string().array(),
origin: z.string().optional().default("api"),
- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
webhook: webhookSchema.optional(),
appendToId: z.string().uuid().optional(),
ignoreInvalidURLs: z.boolean().default(false),
@@ -641,7 +640,7 @@ export const crawlRequestSchema = crawlerOptions
.extend({
url,
origin: z.string().optional().default("api"),
- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
scrapeOptions: baseScrapeOptions.default({}),
webhook: webhookSchema.optional(),
limit: z.number().default(10000),
@@ -673,7 +672,7 @@ export const mapRequestSchema = crawlerOptions
.extend({
url,
origin: z.string().optional().default("api"),
- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
includeSubdomains: z.boolean().default(true),
search: z.string().optional(),
ignoreSitemap: z.boolean().default(false),
@@ -1194,7 +1193,7 @@ export const searchRequestSchema = z
country: z.string().optional().default("us"),
location: z.string().optional(),
origin: z.string().optional().default("api"),
- integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || IntegrationEnum.UNKNOWN),
+ integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
timeout: z.number().int().positive().finite().safe().default(60000),
ignoreInvalidURLs: z.boolean().optional().default(false),
__searchPreviewToken: z.string().optional(),
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 3167e7880..eed92da5e 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -44,7 +44,7 @@ export interface WebScraperOptions {
sitemapped?: boolean;
webhook?: z.infer<typeof webhookSchema>;
v1?: boolean;
- integration?: string;
+ integration?: string | null;
/**
* Disables billing on the worker side.
@@ -95,7 +95,7 @@ export interface FirecrawlJob {
crawlerOptions?: any;
scrapeOptions?: any;
origin: string;
- integration?: string;
+ integration?: string | null;
num_tokens?: number;
retry?: boolean;
crawl_id?: string;