Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolasb98e289f0358
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 061f5c9c8..ab69ca935 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -8,7 +8,28 @@ import {import { getExtractQueue } from "../../services/queue-service";import * as Sentry from "@sentry/node";import { saveExtract } from "../../lib/extract/extract-redis";+import { getTeamIdSyncB } from "../../lib/extract/team-id-sync";+import { performExtraction } from "../../lib/extract/extraction-service";+export async function oldExtract(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>, res: Response<ExtractResponse>, extractId: string){+ // Means that are in the non-queue system+ // TODO: Remove this once all teams have transitioned to the new system+ try {+ const result = await performExtraction(extractId, {+ request: req.body,+ teamId: req.auth.team_id,+ plan: req.auth.plan ?? "free",+ subId: req.acuc?.sub_id ?? undefined,+ });++ return res.status(200).json(result);+ } catch (error) {+ return res.status(500).json({+ success: false,+ error: "Internal server error",+ });+ }+ }/*** Extracts data from the provided URLs based on the request parameters.* Currently in beta.@@ -32,6 +53,10 @@ export async function extractController(extractId,};+ if(await getTeamIdSyncB(req.auth.team_id) && req.body.origin !== "api-sdk") {+ return await oldExtract(req, res, extractId);+ }+await saveExtract(extractId, {id: extractId,team_id: req.auth.team_id,diff --git a/apps/api/src/lib/extract/team-id-sync.ts b/apps/api/src/lib/extract/team-id-sync.tsnew file mode 100644index 000000000..8cf21a141--- /dev/null+++ b/apps/api/src/lib/extract/team-id-sync.ts@@ -0,0 +1,19 @@+import { supabase_service } from "../../services/supabase";+import { logger } from "../logger";++export async function getTeamIdSyncB(teamId: string) {+ try {+ const { data, error } = await supabase_service+ .from("eb-sync")+ .select("team_id")+ .eq("team_id", teamId)+ .limit(1);+ if (error) {+ throw new Error("Error getting team id (sync b)");+ }+ return data[0] ?? null;+ } catch (error) {+ logger.error("Error getting team id (sync b)", error);+ return null;+ }+}diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex bb9fcf6ab..e7e8b65b3 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -884,7 +884,7 @@ export default class FirecrawlApp {try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/extract`,- { ...jsonData, schema: jsonSchema },+ { ...jsonData, schema: jsonSchema, origin: "api-sdk" },headers);diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex 3fff1c4e4..41f8badf6 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -538,7 +538,8 @@ class FirecrawlApp:request_data = {**jsonData,'allowExternalLinks': params.get('allow_external_links', False),- 'schema': schema+ 'schema': schema,+ 'origin': 'api-sdk'}try: