Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: index exploration

Nicolasbf9d41d0b238
diff --git a/apps/api/package.json b/apps/api/package.json
index 00c1bc0e1..670dfc7a6 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -58,6 +58,7 @@
"@devil7softwares/pos": "^1.0.2",
"@dqbd/tiktoken": "^1.0.17",
"@nangohq/node": "^0.40.8",
+ "@pinecone-database/pinecone": "^4.0.0",
"@sentry/cli": "^2.33.1",
"@sentry/node": "^8.26.0",
"@sentry/profiling-node": "^8.26.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 17532d257..082a200f5 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -10,7 +10,7 @@ importers:
dependencies:
'@anthropic-ai/sdk':
specifier: ^0.24.3
- version: 0.24.3
+ version: 0.24.3([email protected])
'@brillout/import':
specifier: ^0.2.2
version: 0.2.3
@@ -29,9 +29,12 @@ importers:
'@nangohq/node':
specifier: ^0.40.8
version: 0.40.8
+ '@pinecone-database/pinecone':
+ specifier: ^4.0.0
+ version: 4.0.0
'@sentry/cli':
specifier: ^2.33.1
- version: 2.33.1
+ version: 2.33.1([email protected])
'@sentry/node':
specifier: ^8.26.0
version: 8.26.0
@@ -79,7 +82,7 @@ importers:
version: 1.1.1
cohere-ai:
specifier: ^7.14.0
- version: 7.14.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))
+ version: 7.14.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))([email protected])
cors:
specifier: ^2.8.5
version: 2.8.5
@@ -130,13 +133,13 @@ importers:
version: 2.9.0
langchain:
specifier: ^0.2.8
languagedetect:
specifier: ^2.0.0
version: 2.0.0
logsnag:
specifier: ^1.0.0
- version: 1.0.0
+ version: 1.0.0([email protected])
luxon:
specifier: ^3.4.3
version: 3.4.4
@@ -157,7 +160,7 @@ importers:
version: 7.0.7(@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected])))([email protected])
openai:
specifier: ^4.57.0
- version: 4.57.0([email protected])
pdf-parse:
specifier: ^1.1.1
version: 1.1.1
@@ -275,7 +278,7 @@ importers:
jest-fetch-mock:
specifier: ^3.0.3
- version: 3.0.3
+ version: 3.0.3([email protected])
mammoth:
specifier: ^1.7.2
version: 1.7.2
@@ -1006,6 +1009,10 @@ packages:
'@pdf-lib/[email protected]':
resolution: {integrity: sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==}
+ '@pinecone-database/[email protected]':
+ resolution: {integrity: sha512-INYS+GBys9v5BRTyn0tv8srVsPTlSRvE3BPE4Wkc/lOEyAIyB9F7DEMXbeF19FOLEgRwCuHTLjzm1niENl+4FA==}
+ engines: {node: '>=18.0.0'}
+
resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
engines: {node: '>=14'}
@@ -2279,6 +2286,9 @@ packages:
resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==}
engines: {node: '>= 0.8'}
+ resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==}
+
resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==}
@@ -4599,7 +4609,7 @@ snapshots:
'@jridgewell/gen-mapping': 0.3.5
'@jridgewell/trace-mapping': 0.3.25
- '@anthropic-ai/[email protected]':
dependencies:
'@types/node': 18.19.39
'@types/node-fetch': 2.6.11
@@ -4607,7 +4617,7 @@ snapshots:
agentkeepalive: 4.5.0
form-data-encoder: 1.7.2
formdata-node: 4.4.1
- node-fetch: 2.7.0
+ node-fetch: 2.7.0([email protected])
web-streams-polyfill: 3.3.3
transitivePeerDependencies:
- encoding
@@ -5577,13 +5587,13 @@ snapshots:
'@jridgewell/resolve-uri': 3.1.2
'@jridgewell/sourcemap-codec': 1.4.15
dependencies:
ansi-styles: 5.2.0
camelcase: 6.3.0
decamelize: 1.2.0
js-tiktoken: 1.0.12
- langsmith: 0.1.34(zyeavx4tfqw3smbbpiinhfxxeu)
+ langsmith: 0.1.34(npkyd6f7wyl3urgrzoxaktl5a4)
ml-distance: 4.0.1
mustache: 4.2.0
p-queue: 6.6.2
@@ -5595,20 +5605,20 @@ snapshots:
- langchain
- openai
dependencies:
js-tiktoken: 1.0.12
- openai: 4.57.0([email protected])
zod: 3.23.8
zod-to-json-schema: 3.23.1([email protected])
transitivePeerDependencies:
- encoding
- langchain
dependencies:
js-tiktoken: 1.0.12
transitivePeerDependencies:
- langchain
@@ -5866,6 +5876,10 @@ snapshots:
dependencies:
pako: 1.0.11
+ '@pinecone-database/[email protected]':
+ dependencies:
+ encoding: 0.1.13
+
optional: true
@@ -5950,10 +5964,10 @@ snapshots:
'@sentry/[email protected]':
optional: true
- '@sentry/[email protected]':
dependencies:
https-proxy-agent: 5.0.1
- node-fetch: 2.7.0
+ node-fetch: 2.7.0([email protected])
progress: 2.0.3
proxy-from-env: 1.1.0
which: 2.0.2
@@ -7088,7 +7102,7 @@ snapshots:
dependencies:
'@aws-sdk/client-sagemaker': 3.679.0
'@aws-sdk/credential-providers': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))
@@ -7098,7 +7112,7 @@ snapshots:
form-data-encoder: 4.0.2
formdata-node: 6.0.3
js-base64: 3.7.2
- node-fetch: 2.7.0
+ node-fetch: 2.7.0([email protected])
qs: 6.11.2
readable-stream: 4.5.2
url-join: 4.0.1
@@ -7208,9 +7222,9 @@ snapshots:
dependencies:
luxon: 3.4.4
dependencies:
- node-fetch: 2.7.0
+ node-fetch: 2.7.0([email protected])
transitivePeerDependencies:
- encoding
@@ -7365,6 +7379,10 @@ snapshots:
+ dependencies:
+ iconv-lite: 0.6.3
+
dependencies:
once: 1.4.0
@@ -7899,9 +7917,9 @@ snapshots:
dependencies:
- node-fetch: 2.7.0
+ node-fetch: 2.7.0([email protected])
whatwg-fetch: 3.6.20
transitivePeerDependencies:
- encoding
@@ -8070,9 +8088,9 @@ snapshots:
jest-mock: 29.7.0
jest-util: 29.7.0
dependencies:
- cross-fetch: 3.1.8
+ cross-fetch: 3.1.8([email protected])
promise-polyfill: 8.3.0
transitivePeerDependencies:
- encoding
@@ -8342,17 +8360,17 @@ snapshots:
dependencies:
binary-extensions: 2.3.0
js-tiktoken: 1.0.12
js-yaml: 4.1.0
jsonpointer: 5.0.1
langchainhub: 0.0.11
- langsmith: 0.1.34(zyeavx4tfqw3smbbpiinhfxxeu)
+ langsmith: 0.1.34(npkyd6f7wyl3urgrzoxaktl5a4)
ml-distance: 4.0.1
openapi-types: 12.1.3
p-retry: 4.6.2
@@ -8362,6 +8380,7 @@ snapshots:
zod-to-json-schema: 3.23.1([email protected])
optionalDependencies:
'@aws-sdk/credential-provider-node': 3.679.0(@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])
+ '@pinecone-database/pinecone': 4.0.0
'@supabase/supabase-js': 2.44.2
axios: 1.7.2
cheerio: 1.0.0-rc.12
@@ -8381,7 +8400,7 @@ snapshots:
- [email protected](zyeavx4tfqw3smbbpiinhfxxeu):
+ [email protected](npkyd6f7wyl3urgrzoxaktl5a4):
dependencies:
'@types/uuid': 9.0.8
commander: 10.0.1
@@ -8390,9 +8409,9 @@ snapshots:
p-retry: 4.6.2
uuid: 9.0.1
optionalDependencies:
- openai: 4.57.0([email protected])
@@ -8442,9 +8461,9 @@ snapshots:
dependencies:
- isomorphic-fetch: 3.0.0
+ isomorphic-fetch: 3.0.0([email protected])
transitivePeerDependencies:
- encoding
@@ -8703,9 +8722,11 @@ snapshots:
dependencies:
whatwg-url: 5.0.0
+ optionalDependencies:
+ encoding: 0.1.13
dependencies:
@@ -8780,7 +8801,7 @@ snapshots:
transitivePeerDependencies:
- debug
dependencies:
'@types/node': 18.19.39
'@types/node-fetch': 2.6.11
@@ -8789,7 +8810,7 @@ snapshots:
agentkeepalive: 4.5.0
form-data-encoder: 1.7.2
formdata-node: 4.4.1
- node-fetch: 2.7.0
+ node-fetch: 2.7.0([email protected])
qs: 6.12.2
optionalDependencies:
zod: 3.23.8
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index adc080f2b..d47692839 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -17,6 +17,7 @@ import expressWs from "express-ws";
import { ErrorResponse, ResponseWithSentry } from "./controllers/v1/types";
import { ZodError } from "zod";
import { v4 as uuidv4 } from "uuid";
+import { searchSimilarPages } from "./lib/extract/index/pinecone";
const { createBullBoard } = require("@bull-board/api");
const { BullAdapter } = require("@bull-board/api/bullAdapter");
diff --git a/apps/api/src/lib/extract/index/pinecone.ts b/apps/api/src/lib/extract/index/pinecone.ts
new file mode 100644
index 000000000..603cd38f1
--- /dev/null
+++ b/apps/api/src/lib/extract/index/pinecone.ts
@@ -0,0 +1,141 @@
+import { Pinecone } from '@pinecone-database/pinecone';
+import { Document } from '../../../controllers/v1/types';
+import { logger } from '../../logger';
+import OpenAI from "openai";
+
+const openai = new OpenAI({
+ apiKey: process.env.OPENAI_API_KEY,
+});
+
+const pinecone = new Pinecone({
+ apiKey: process.env.PINECONE_API_KEY!,
+});
+
+const INDEX_NAME = process.env.PINECONE_INDEX_NAME ?? "";
+
+export interface PageMetadata {
+ url: string;
+ originUrl: string;
+ title?: string;
+ description?: string;
+ crawlId?: string;
+ teamId?: string;
+ timestamp: number;
+ markdown?: string;
+}
+
+async function getEmbedding(text: string) {
+ const embedding = await openai.embeddings.create({
+ model: "text-embedding-3-small",
+ input: text,
+ encoding_format: "float",
+ });
+
+ return embedding.data[0].embedding;
+}
+
+function normalizeUrl(url: string) {
+ const urlO = new URL(url);
+ if (!urlO.hostname.startsWith("www.")) {
+ urlO.hostname = "www." + urlO.hostname;
+ }
+ return urlO.href;
+}
+
+export async function indexPage(
+ document: Document,
+ originUrl: string,
+ crawlId?: string,
+ teamId?: string
+) {
+ try {
+ const index = pinecone.index(INDEX_NAME);
+
+ // Create text to embed
+ const textToEmbed = [
+ document.metadata.title,
+ document.metadata.description,
+ document.markdown
+ ].filter(Boolean).join('\n\n');
+
+ // Get embedding from OpenAI
+ const embedding = await getEmbedding(textToEmbed);
+
+ // Prepare metadata
+ const metadata: PageMetadata = {
+ url: normalizeUrl(document.metadata.sourceURL || document.metadata.url!),
+ originUrl: normalizeUrl(originUrl),
+ title: document.metadata.title,
+ description: document.metadata.description,
+ crawlId,
+ teamId,
+ markdown: document.markdown,
+ timestamp: Date.now()
+ };
+
+ // Upsert to Pinecone
+ await index.upsert([{
+ id: document.metadata.sourceURL || document.metadata.url!,
+ values: embedding,
+ metadata: {
+ ...metadata,
+ [document.metadata.sourceURL || document.metadata.url!]: true
+ }
+ }]);
+
+ logger.debug('Successfully indexed page in Pinecone', {
+ url: metadata.url,
+ crawlId
+ });
+
+ } catch (error) {
+ logger.error('Failed to index page in Pinecone', {
+ error,
+ url: document.metadata.sourceURL || document.metadata.url,
+ crawlId
+ });
+ }
+}
+
+export async function searchSimilarPages(
+ query: string,
+ originUrl?: string,
+ limit: number = 10
+) {
+ try {
+ const index = pinecone.index(INDEX_NAME);
+
+ // Get query embedding from OpenAI
+ const queryEmbedding = await getEmbedding(query);
+
+ const queryParams: any = {
+ vector: queryEmbedding,
+ topK: limit,
+ includeMetadata: true
+ };
+
+ // Add filter if originUrl is provided
+ if (originUrl) {
+ queryParams.filter = {
+ [originUrl]: { $contains: normalizeUrl(originUrl) }
+ };
+ }
+
+ const results = await index.query(queryParams);
+ return results.matches.map(match => ({
+ url: match.metadata?.url,
+ title: match.metadata?.title,
+ description: match.metadata?.description,
+ score: match.score,
+ markdown: match.metadata?.markdown
+ }));
+
+ } catch (error) {
+ logger.error('Failed to search similar pages in Pinecone', {
+ error,
+ query,
+ originUrl
+ });
+ return [];
+ }
+}
diff --git a/apps/api/src/lib/ranker.ts b/apps/api/src/lib/ranker.ts
index bffbc9c27..02d59457c 100644
--- a/apps/api/src/lib/ranker.ts
+++ b/apps/api/src/lib/ranker.ts
@@ -10,7 +10,7 @@ const openai = new OpenAI({
async function getEmbedding(text: string) {
const embedding = await openai.embeddings.create({
- model: "text-embedding-ada-002",
+ model: "text-embedding-3-small",
input: text,
encoding_format: "float",
});