Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

remove indexes that are no longer used

Gergő Móricz7a8be132204b
diff --git a/apps/api/package.json b/apps/api/package.json
index 2461df18c..9866c3bc8 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -69,7 +69,6 @@
"@google-cloud/storage": "^7.16.0",
"@nangohq/node": "^0.40.8",
"@openrouter/ai-sdk-provider": "^0.4.5",
- "@pinecone-database/pinecone": "^4.0.0",
"@sentry/cli": "^2.33.1",
"@sentry/node": "^8.26.0",
"@sentry/profiling-node": "^8.26.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 333cd85c0..d394f007e 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -59,9 +59,6 @@ importers:
'@openrouter/ai-sdk-provider':
specifier: ^0.4.5
version: 0.4.5([email protected])
- '@pinecone-database/pinecone':
- specifier: ^4.0.0
- version: 4.0.0
'@sentry/cli':
specifier: ^2.33.1
version: 2.33.1([email protected])
@@ -1232,10 +1229,6 @@ packages:
'@pdf-lib/[email protected]':
resolution: {integrity: sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==}
- '@pinecone-database/[email protected]':
- resolution: {integrity: sha512-INYS+GBys9v5BRTyn0tv8srVsPTlSRvE3BPE4Wkc/lOEyAIyB9F7DEMXbeF19FOLEgRwCuHTLjzm1niENl+4FA==}
- engines: {node: '>=18.0.0'}
-
resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
engines: {node: '>=14'}
@@ -6331,10 +6324,6 @@ snapshots:
dependencies:
pako: 1.0.11
- '@pinecone-database/[email protected]':
- dependencies:
- encoding: 0.1.13
-
optional: true
@@ -7921,6 +7910,7 @@ snapshots:
dependencies:
iconv-lite: 0.6.3
+ optional: true
dependencies:
diff --git a/apps/api/src/lib/extract/index/pinecone.ts b/apps/api/src/lib/extract/index/pinecone.ts
deleted file mode 100644
index 44aed9600..000000000
--- a/apps/api/src/lib/extract/index/pinecone.ts
+++ /dev/null
@@ -1,164 +0,0 @@
-import { Pinecone } from "@pinecone-database/pinecone";
-import { Document } from "../../../controllers/v1/types";
-import { logger } from "../../logger";
-import { embed } from "ai";
-import { getEmbeddingModel } from "../../generic-ai";
-
-const pinecone = new Pinecone({
- apiKey: process.env.PINECONE_API_KEY!,
-});
-
-const INDEX_NAME = process.env.PINECONE_INDEX_NAME ?? "";
-
-const MAX_METADATA_SIZE = 30 * 1024; // 30KB in bytes
-
-export interface PageMetadata {
- url: string;
- originUrl: string;
- title?: string;
- description?: string;
- crawlId?: string;
- teamId?: string;
- timestamp: number;
- markdown?: string;
-}
-
-async function getEmbedding(text: string) {
- const { embedding } = await embed({
- model: getEmbeddingModel("text-embedding-3-small"),
- value: text,
- });
-
- return embedding;
-}
-
-function normalizeUrl(url: string) {
- const urlO = new URL(url);
- if (!urlO.hostname.startsWith("www.")) {
- urlO.hostname = "www." + urlO.hostname;
- }
- return urlO.href;
-}
-
-export async function indexPage({
- document,
- originUrl,
- crawlId,
- teamId,
-}: {
- document: Document;
- originUrl: string;
- crawlId?: string;
- teamId?: string;
-}) {
- try {
- const index = pinecone.index(INDEX_NAME);
-
- // Trim markdown if it's too long
- let trimmedMarkdown = document.markdown;
- if (
- trimmedMarkdown &&
- Buffer.byteLength(trimmedMarkdown, "utf-8") > MAX_METADATA_SIZE
- ) {
- trimmedMarkdown = trimmedMarkdown.slice(
- 0,
- Math.floor(MAX_METADATA_SIZE / 2),
- ); // Using half the size to be safe with UTF-8 encoding
- }
-
- // Create text to embed
- const textToEmbed = [
- document.metadata.title,
- document.metadata.description,
- trimmedMarkdown,
- ]
- .filter(Boolean)
- .join("\n\n");
-
- // Get embedding from OpenAI
- const embedding = await getEmbedding(textToEmbed);
-
- const normalizedUrl = normalizeUrl(
- document.metadata.sourceURL || document.metadata.url!,
- );
-
- // Prepare metadata
- const metadata: PageMetadata = {
- url: normalizedUrl,
- originUrl: normalizeUrl(originUrl),
- title: document.metadata.title ?? document.metadata.ogTitle ?? "",
- description:
- document.metadata.description ?? document.metadata.ogDescription ?? "",
- crawlId,
- teamId,
- markdown: trimmedMarkdown,
- timestamp: Date.now(),
- };
-
- // Upsert to Pinecone
- await index.upsert([
- {
- id: normalizedUrl,
- values: embedding,
- metadata: {
- ...metadata,
- [document.metadata.sourceURL || document.metadata.url!]: true,
- },
- },
- ]);
-
- logger.debug("Successfully indexed page in Pinecone", {
- url: metadata.url,
- crawlId,
- });
- } catch (error) {
- logger.error("Failed to index page in Pinecone", {
- error,
- url: document.metadata.sourceURL || document.metadata.url,
- crawlId,
- });
- }
-}
-
-export async function searchSimilarPages(
- query: string,
- originUrl?: string,
- limit: number = 1000,
-): Promise<any[]> {
- try {
- const index = pinecone.index(INDEX_NAME);
-
- // Get query embedding from OpenAI
- const queryEmbedding = await getEmbedding(query);
-
- const queryParams: any = {
- vector: queryEmbedding,
- topK: limit,
- includeMetadata: true,
- };
-
- const normalizedOriginUrl = originUrl ? normalizeUrl(originUrl) : undefined;
- // Add filter if originUrl is provided
- if (normalizedOriginUrl) {
- queryParams.filter = {
- originUrl: { $eq: normalizedOriginUrl },
- };
- }
-
- const results = await index.query(queryParams);
- return results.matches.map((match) => ({
- url: match.metadata?.url,
- title: match.metadata?.title,
- description: match.metadata?.description,
- score: match.score,
- markdown: match.metadata?.markdown,
- }));
- } catch (error) {
- logger.error("Failed to search similar pages in Pinecone", {
- error,
- query,
- originUrl,
- });
- return [];
- }
-}
diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.ts
index 62b1d2662..6ff6e77cb 100644
--- a/apps/api/src/lib/extract/reranker.ts
+++ b/apps/api/src/lib/extract/reranker.ts
@@ -4,7 +4,6 @@ import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { logger } from "../logger";
import { CohereClient } from "cohere-ai";
import { extractConfig } from "./config";
-import { searchSimilarPages } from "./index/pinecone";
import { generateCompletions } from "../../scraper/scrapeURL/transformers/llmExtract";
import { buildRerankerUserPrompt } from "./build-prompts";
import { buildRerankerSystemPrompt } from "./build-prompts";
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index 2f9b6495a..cddba97b1 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -15,7 +15,6 @@ import {
ScrapeUrlResponse,
} from "../scraper/scrapeURL";
import { Engine } from "../scraper/scrapeURL/engines";
-import { indexPage } from "../lib/extract/index/pinecone";
import { CostTracking } from "../lib/extract/extraction-service";
configDotenv();
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index e73afeb52..162ddb43b 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -21,47 +21,6 @@ function cleanOfNull<T>(x: T): T {
}
}
-async function indexJob(job: FirecrawlJob): Promise<void> {
- try {
- if (job.mode !== "single_urls" && job.mode !== "scrape") {
- return;
- }
-
- const response = await fetch(`${process.env.FIRE_INDEX_SERVER_URL}/api/jobs`, {
- method: 'POST',
- headers: {
- 'Content-Type': 'application/json',
- },
- body: JSON.stringify({
- url: job.url,
- mode: job.mode || "scrape",
- docs: job.docs,
- origin: job.origin,
- success: job.success,
- time_taken: job.time_taken,
- num_tokens: job.num_tokens,
- page_options: job.scrapeOptions,
- date_added: new Date().toISOString(),
- }),
- });
-
- if (!response.ok) {
- const errorData = await response.json();
- // logger.error(`Failed to send job to external server: ${response.status} ${response.statusText}`, {
- // error: errorData,
- // scrapeId: job.job_id,
- // });
- } else {
- // logger.debug("Job sent to external server successfully!", { scrapeId: job.job_id });
- }
- } catch (error) {
- // logger.error(`Error sending job to external server: ${error.message}`, {
- // error,
- // scrapeId: job.job_id,
- // });
- }
-}
-
export async function logJob(job: FirecrawlJob, force: boolean = false) {
try {
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
@@ -106,11 +65,6 @@ export async function logJob(job: FirecrawlJob, force: boolean = false) {
pdf_num_pages: job.pdf_num_pages ?? null,
};
- // Send job to external server
- if (process.env.FIRE_INDEX_SERVER_URL) {
- indexJob(job);
- }
-
if (process.env.GCS_BUCKET_NAME) {
await saveJobToGCS(job);
}
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 9624cfdac..c007e18b6 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -61,7 +61,6 @@ import {
} from "../lib/concurrency-limit";
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";
-import { indexPage } from "../lib/extract/index/pinecone";
import { Document } from "../controllers/v1/types";
import {
ExtractResult,
@@ -1045,25 +1044,6 @@ async function processKickoffJob(job: Job & { id: string }, token: string) {
}
}
-async function indexJob(job: Job & { id: string }, document: Document) {
- if (
- document &&
- document.markdown &&
- job.data.team_id === process.env.BACKGROUND_INDEX_TEAM_ID!
- ) {
- // indexPage({
- // document: document,
- // originUrl: job.data.crawl_id
- // ? (await getCrawl(job.data.crawl_id))?.originUrl!
- // : document.metadata.sourceURL!,
- // crawlId: job.data.crawl_id,
- // teamId: job.data.team_id,
- // }).catch((error) => {
- // _logger.error("Error indexing page", { error });
- // });
- }
-}
-
async function processJob(job: Job & { id: string }, token: string) {
const logger = _logger.child({
module: "queue-worker",
@@ -1262,8 +1242,6 @@ async function processJob(job: Job & { id: string }, token: string) {
);
}
- indexJob(job, doc);
-
logger.debug("Declaring job as done...");
await addCrawlJobDone(job.data.crawl_id, job.id, true);
@@ -1380,8 +1358,6 @@ async function processJob(job: Job & { id: string }, token: string) {
cost_tracking: costTracking,
pdf_num_pages: doc.metadata.numPages,
});
-
- indexJob(job, doc);
}
if (job.data.is_scrape !== true) {