Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge branch 'main' into nsc/extract-queue
Nicolasbb2759444311
diff --cc apps/api/src/services/queue-worker.tsindex c235a87bd,a48c798b4..422a8160d--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@@ -52,7 -50,8 +52,9 @@@ import { isUrlBlocked } from "../scrapeimport { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";import { indexPage } from "../lib/extract/index/pinecone";import { Document } from "../controllers/v1/types";+import { performExtraction } from "../lib/extract/extraction-service";+ import { supabase_service } from "../services/supabase";+ import { normalizeUrl, normalizeUrlOnlyHostname } from "../lib/canonical-url";configDotenv();