Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/precrawl): insert request rows
Gergő Móricz4885fd7e15c5
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 5d34d9b1d..b39bdeac3 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -42,6 +42,7 @@ import { crawlGroup } from "../worker/nuq";import { getACUCTeam } from "../../controllers/auth";import { supabase_service } from "../supabase";import { processEngpickerJob } from "../../lib/engpicker";+import { logRequest } from "../logging/log_job";const workerLockDuration = config.WORKER_LOCK_DURATION;const workerStalledCheckInterval = config.WORKER_STALLED_CHECK_INTERVAL;@@ -447,6 +448,18 @@ const processPrecrawlJob = async (token: string, job: Job) => {try {const { url, budget: limit } = target;+ const crawlId = uuidv7();+ await logRequest({+ id: crawlId,+ kind: "crawl",+ api_version: "v2",+ team_id: teamId,+ origin: "precrawl",+ target_hint: url,+ zeroDataRetention: false,+ api_key_id: null,+ });+const crawlerOptions = {...crawlRequestSchema.parse({ url, limit }),url: undefined, // unsure why this is needed but leaving for now@@ -476,8 +489,6 @@ const processPrecrawlJob = async (token: string, job: Job) => {zeroDataRetention: false,};- const crawlId = uuidv7();-await crawlGroup.addGroup(crawlId,sc.team_id,