Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
remove indexes that are no longer used
Gergő Móricz7a8be132204b
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 2461df18c..9866c3bc8 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -69,7 +69,6 @@"@google-cloud/storage": "^7.16.0","@nangohq/node": "^0.40.8","@openrouter/ai-sdk-provider": "^0.4.5",- "@pinecone-database/pinecone": "^4.0.0","@sentry/cli": "^2.33.1","@sentry/node": "^8.26.0","@sentry/profiling-node": "^8.26.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 333cd85c0..d394f007e 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -59,9 +59,6 @@ importers:'@openrouter/ai-sdk-provider':specifier: ^0.4.5version: 0.4.5([email protected])- '@pinecone-database/pinecone':- specifier: ^4.0.0- version: 4.0.0'@sentry/cli':specifier: ^2.33.1version: 2.33.1([email protected])@@ -1232,10 +1229,6 @@ packages:'@pdf-lib/[email protected]':resolution: {integrity: sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==}- '@pinecone-database/[email protected]':- resolution: {integrity: sha512-INYS+GBys9v5BRTyn0tv8srVsPTlSRvE3BPE4Wkc/lOEyAIyB9F7DEMXbeF19FOLEgRwCuHTLjzm1niENl+4FA==}- engines: {node: '>=18.0.0'}-'@pkgjs/[email protected]':resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}engines: {node: '>=14'}@@ -6331,10 +6324,6 @@ snapshots:dependencies:pako: 1.0.11- '@pinecone-database/[email protected]':- dependencies:- encoding: 0.1.13-'@pkgjs/[email protected]':optional: true@@ -7921,6 +7910,7 @@ snapshots:dependencies:iconv-lite: 0.6.3+ optional: truedependencies:diff --git a/apps/api/src/lib/extract/index/pinecone.ts b/apps/api/src/lib/extract/index/pinecone.tsdeleted file mode 100644index 44aed9600..000000000--- a/apps/api/src/lib/extract/index/pinecone.ts+++ /dev/null@@ -1,164 +0,0 @@-import { Pinecone } from "@pinecone-database/pinecone";-import { Document } from "../../../controllers/v1/types";-import { logger } from "../../logger";-import { embed } from "ai";-import { getEmbeddingModel } from "../../generic-ai";--const pinecone = new Pinecone({- apiKey: process.env.PINECONE_API_KEY!,-});--const INDEX_NAME = process.env.PINECONE_INDEX_NAME ?? "";--const MAX_METADATA_SIZE = 30 * 1024; // 30KB in bytes--export interface PageMetadata {- url: string;- originUrl: string;- title?: string;- description?: string;- crawlId?: string;- teamId?: string;- timestamp: number;- markdown?: string;-}--async function getEmbedding(text: string) {- const { embedding } = await embed({- model: getEmbeddingModel("text-embedding-3-small"),- value: text,- });-- return embedding;-}--function normalizeUrl(url: string) {- const urlO = new URL(url);- if (!urlO.hostname.startsWith("www.")) {- urlO.hostname = "www." + urlO.hostname;- }- return urlO.href;-}--export async function indexPage({- document,- originUrl,- crawlId,- teamId,-}: {- document: Document;- originUrl: string;- crawlId?: string;- teamId?: string;-}) {- try {- const index = pinecone.index(INDEX_NAME);-- // Trim markdown if it's too long- let trimmedMarkdown = document.markdown;- if (- trimmedMarkdown &&- Buffer.byteLength(trimmedMarkdown, "utf-8") > MAX_METADATA_SIZE- ) {- trimmedMarkdown = trimmedMarkdown.slice(- 0,- Math.floor(MAX_METADATA_SIZE / 2),- ); // Using half the size to be safe with UTF-8 encoding- }-- // Create text to embed- const textToEmbed = [- document.metadata.title,- document.metadata.description,- trimmedMarkdown,- ]- .filter(Boolean)- .join("\n\n");-- // Get embedding from OpenAI- const embedding = await getEmbedding(textToEmbed);-- const normalizedUrl = normalizeUrl(- document.metadata.sourceURL || document.metadata.url!,- );-- // Prepare metadata- const metadata: PageMetadata = {- url: normalizedUrl,- originUrl: normalizeUrl(originUrl),- title: document.metadata.title ?? document.metadata.ogTitle ?? "",- description:- document.metadata.description ?? document.metadata.ogDescription ?? "",- crawlId,- teamId,- markdown: trimmedMarkdown,- timestamp: Date.now(),- };-- // Upsert to Pinecone- await index.upsert([- {- id: normalizedUrl,- values: embedding,- metadata: {- ...metadata,- [document.metadata.sourceURL || document.metadata.url!]: true,- },- },- ]);-- logger.debug("Successfully indexed page in Pinecone", {- url: metadata.url,- crawlId,- });- } catch (error) {- logger.error("Failed to index page in Pinecone", {- error,- url: document.metadata.sourceURL || document.metadata.url,- crawlId,- });- }-}--export async function searchSimilarPages(- query: string,- originUrl?: string,- limit: number = 1000,-): Promise<any[]> {- try {- const index = pinecone.index(INDEX_NAME);-- // Get query embedding from OpenAI- const queryEmbedding = await getEmbedding(query);-- const queryParams: any = {- vector: queryEmbedding,- topK: limit,- includeMetadata: true,- };-- const normalizedOriginUrl = originUrl ? normalizeUrl(originUrl) : undefined;- // Add filter if originUrl is provided- if (normalizedOriginUrl) {- queryParams.filter = {- originUrl: { $eq: normalizedOriginUrl },- };- }-- const results = await index.query(queryParams);- return results.matches.map((match) => ({- url: match.metadata?.url,- title: match.metadata?.title,- description: match.metadata?.description,- score: match.score,- markdown: match.metadata?.markdown,- }));- } catch (error) {- logger.error("Failed to search similar pages in Pinecone", {- error,- query,- originUrl,- });- return [];- }-}diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.tsindex 62b1d2662..6ff6e77cb 100644--- a/apps/api/src/lib/extract/reranker.ts+++ b/apps/api/src/lib/extract/reranker.ts@@ -4,7 +4,6 @@ import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { logger } from "../logger";import { CohereClient } from "cohere-ai";import { extractConfig } from "./config";-import { searchSimilarPages } from "./index/pinecone";import { generateCompletions } from "../../scraper/scrapeURL/transformers/llmExtract";import { buildRerankerUserPrompt } from "./build-prompts";import { buildRerankerSystemPrompt } from "./build-prompts";diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 2f9b6495a..cddba97b1 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -15,7 +15,6 @@ import {ScrapeUrlResponse,} from "../scraper/scrapeURL";import { Engine } from "../scraper/scrapeURL/engines";-import { indexPage } from "../lib/extract/index/pinecone";import { CostTracking } from "../lib/extract/extraction-service";configDotenv();diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex e73afeb52..162ddb43b 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -21,47 +21,6 @@ function cleanOfNull<T>(x: T): T {}}-async function indexJob(job: FirecrawlJob): Promise<void> {- try {- if (job.mode !== "single_urls" && job.mode !== "scrape") {- return;- }-- const response = await fetch(`${process.env.FIRE_INDEX_SERVER_URL}/api/jobs`, {- method: 'POST',- headers: {- 'Content-Type': 'application/json',- },- body: JSON.stringify({- url: job.url,- mode: job.mode || "scrape",- docs: job.docs,- origin: job.origin,- success: job.success,- time_taken: job.time_taken,- num_tokens: job.num_tokens,- page_options: job.scrapeOptions,- date_added: new Date().toISOString(),- }),- });-- if (!response.ok) {- const errorData = await response.json();- // logger.error(`Failed to send job to external server: ${response.status} ${response.statusText}`, {- // error: errorData,- // scrapeId: job.job_id,- // });- } else {- // logger.debug("Job sent to external server successfully!", { scrapeId: job.job_id });- }- } catch (error) {- // logger.error(`Error sending job to external server: ${error.message}`, {- // error,- // scrapeId: job.job_id,- // });- }-}-export async function logJob(job: FirecrawlJob, force: boolean = false) {try {const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";@@ -106,11 +65,6 @@ export async function logJob(job: FirecrawlJob, force: boolean = false) {pdf_num_pages: job.pdf_num_pages ?? null,};- // Send job to external server- if (process.env.FIRE_INDEX_SERVER_URL) {- indexJob(job);- }-if (process.env.GCS_BUCKET_NAME) {await saveJobToGCS(job);}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 9624cfdac..c007e18b6 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -61,7 +61,6 @@ import {} from "../lib/concurrency-limit";import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";-import { indexPage } from "../lib/extract/index/pinecone";import { Document } from "../controllers/v1/types";import {ExtractResult,@@ -1045,25 +1044,6 @@ async function processKickoffJob(job: Job & { id: string }, token: string) {}}-async function indexJob(job: Job & { id: string }, document: Document) {- if (- document &&- document.markdown &&- job.data.team_id === process.env.BACKGROUND_INDEX_TEAM_ID!- ) {- // indexPage({- // document: document,- // originUrl: job.data.crawl_id- // ? (await getCrawl(job.data.crawl_id))?.originUrl!- // : document.metadata.sourceURL!,- // crawlId: job.data.crawl_id,- // teamId: job.data.team_id,- // }).catch((error) => {- // _logger.error("Error indexing page", { error });- // });- }-}-async function processJob(job: Job & { id: string }, token: string) {const logger = _logger.child({module: "queue-worker",@@ -1262,8 +1242,6 @@ async function processJob(job: Job & { id: string }, token: string) {);}- indexJob(job, doc);-logger.debug("Declaring job as done...");await addCrawlJobDone(job.data.crawl_id, job.id, true);@@ -1380,8 +1358,6 @@ async function processJob(job: Job & { id: string }, token: string) {cost_tracking: costTracking,pdf_num_pages: doc.metadata.numPages,});-- indexJob(job, doc);}if (job.data.is_scrape !== true) {