Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix unused files

Gergő Móricz3d2ddbf09a2d
diff --git a/apps/api/audit-ci.jsonc b/apps/api/audit-ci.jsonc
index 473dfcdab..8c59c3718 100644
--- a/apps/api/audit-ci.jsonc
+++ b/apps/api/audit-ci.jsonc
@@ -6,6 +6,7 @@
"GHSA-3gc7-fjrx-p6mg|@coinbase/x402>@coinbase/cdp-sdk>@solana/spl-token>@solana/buffer-layout-utils>bigint-buffer", // never gets called in our code paths
"GHSA-qj3p-xc97-xw74|@coinbase/x402>x402>wagmi>@wagmi/connectors>@metamask/sdk", // we don't use the Metamask SDK in the browser
"GHSA-qj3p-xc97-xw74|@coinbase/x402>x402>wagmi>@wagmi/connectors>@metamask/sdk>@metamask/sdk-communication-layer", // we don't use the Metamask SDK in the browser
- "GHSA-ffrw-9mx8-89p8|@coinbase/x402>x402>wagmi>@wagmi/connectors>@walletconnect/ethereum-provider>@reown/appkit>@reown/appkit-utils>@walletconnect/logger>pino>fast-redact" // unused
+ "GHSA-ffrw-9mx8-89p8|@coinbase/x402>x402>wagmi>@wagmi/connectors>@walletconnect/ethereum-provider>@reown/appkit>@reown/appkit-utils>@walletconnect/logger>pino>fast-redact", // unused
+ "GHSA-m732-5p4w-x69g|@coinbase/x402>x402>wagmi>@wagmi/connectors>porto>hono" // we don't use hono
]
-}
\ No newline at end of file
+}
diff --git a/apps/api/package.json b/apps/api/package.json
index 0fd875f51..404e738e2 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -87,7 +87,6 @@
"@opentelemetry/core": "^2.1.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.205.0",
"@opentelemetry/sdk-node": "^0.205.0",
- "@pinecone-database/pinecone": "^6.1.2",
"@sentry/cli": "^2.33.1",
"@sentry/node": "^9.40.0",
"@sentry/profiling-node": "^9.40.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 0bbbc7abb..65d704edc 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -76,9 +76,6 @@ importers:
'@opentelemetry/sdk-node':
specifier: ^0.205.0
version: 0.205.0(@opentelemetry/[email protected])
- '@pinecone-database/pinecone':
- specifier: ^6.1.2
- version: 6.1.2
'@sentry/cli':
specifier: ^2.33.1
version: 2.33.1([email protected])
@@ -2626,10 +2623,6 @@ packages:
resolution: {integrity: sha512-IHnV6A+zxU7XwmKFinmYjUcwlyK9+xkG3/s9KcQhI9BjQKycrJ1JRO+FbNYPwZiPKW3je/DR0k7w8/gLa5eaxQ==}
deprecated: 'The package is now available as "qr": npm install qr'
- '@pinecone-database/[email protected]':
- resolution: {integrity: sha512-ydIlbtgIIHFgBL08sPzua5ckmOgtjgDz8xg21CnP1fqnnEgDmOlnfd10MRKU+fvFRhDlh4Md37SwZDr0d4cBqg==}
- engines: {node: '>=18.0.0'}
-
resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
engines: {node: '>=14'}
@@ -7537,7 +7530,7 @@ snapshots:
idb-keyval: 6.2.1
preact: 10.24.2
transitivePeerDependencies:
- '@types/react'
@@ -7611,7 +7604,7 @@ snapshots:
idb-keyval: 6.2.1
preact: 10.24.2
transitivePeerDependencies:
- '@types/react'
@@ -7814,11 +7807,11 @@ snapshots:
ethereum-cryptography: 2.2.1
micro-ftch: 0.3.1
dependencies:
'@metamask/rpc-errors': 7.0.2
eventemitter3: 5.0.1
transitivePeerDependencies:
- supports-color
@@ -9802,8 +9795,6 @@ snapshots:
'@paulmillr/[email protected]': {}
- '@pinecone-database/[email protected]': {}
-
optional: true
@@ -9860,7 +9851,7 @@ snapshots:
dependencies:
big.js: 6.2.2
dayjs: 1.11.13
transitivePeerDependencies:
- bufferutil
- typescript
@@ -9873,7 +9864,7 @@ snapshots:
'@reown/appkit-wallet': 1.7.8([email protected])([email protected])([email protected])
valtio: 1.13.2([email protected])
transitivePeerDependencies:
- '@azure/app-configuration'
- '@azure/cosmos'
@@ -10019,7 +10010,7 @@ snapshots:
'@walletconnect/logger': 2.1.2
valtio: 1.13.2([email protected])
transitivePeerDependencies:
- '@azure/app-configuration'
- '@azure/cosmos'
@@ -10072,7 +10063,7 @@ snapshots:
bs58: 6.0.0
valtio: 1.13.2([email protected])
transitivePeerDependencies:
- '@azure/app-configuration'
- '@azure/cosmos'
@@ -10113,7 +10104,7 @@ snapshots:
dependencies:
'@safe-global/safe-gateway-typescript-sdk': 3.23.1
transitivePeerDependencies:
- bufferutil
- typescript
@@ -11072,19 +11063,19 @@ snapshots:
dependencies:
'@types/yargs-parser': 21.0.3
dependencies:
'@safe-global/safe-apps-provider': 0.18.6([email protected])([email protected])([email protected])([email protected])
cbw-sdk: '@coinbase/[email protected]'
optionalDependencies:
typescript: 5.8.3
transitivePeerDependencies:
@@ -11118,11 +11109,11 @@ snapshots:
- wagmi
- zod
dependencies:
eventemitter3: 5.0.1
mipd: 0.0.7([email protected])
optionalDependencies:
'@tanstack/query-core': 5.83.1
@@ -11670,6 +11661,11 @@ snapshots:
typescript: 5.8.3
zod: 3.22.4
+ optionalDependencies:
+ typescript: 5.8.3
+ zod: 3.24.2
+
optionalDependencies:
typescript: 5.8.3
@@ -11680,6 +11676,11 @@ snapshots:
typescript: 5.8.3
zod: 3.22.4
+ optionalDependencies:
+ typescript: 5.8.3
+ zod: 3.24.2
+
optionalDependencies:
typescript: 5.8.3
@@ -14151,6 +14152,21 @@ snapshots:
transitivePeerDependencies:
- zod
+ dependencies:
+ '@adraffy/ens-normalize': 1.11.1
+ '@noble/ciphers': 1.3.0
+ '@noble/curves': 1.9.6
+ '@noble/hashes': 1.8.0
+ '@scure/bip32': 1.7.0
+ '@scure/bip39': 1.6.0
+ eventemitter3: 5.0.1
+ optionalDependencies:
+ typescript: 5.8.3
+ transitivePeerDependencies:
+ - zod
+
dependencies:
'@adraffy/ens-normalize': 1.11.1
@@ -14390,21 +14406,21 @@ snapshots:
dependencies:
hono: 4.9.10
idb-keyval: 6.2.2
mipd: 0.0.7([email protected])
zod: 4.1.11
optionalDependencies:
'@tanstack/react-query': 5.84.1([email protected])
react: 18.3.1
typescript: 5.8.3
transitivePeerDependencies:
- '@types/react'
- immer
@@ -15334,6 +15350,23 @@ snapshots:
- utf-8-validate
- zod
+ dependencies:
+ '@noble/curves': 1.9.2
+ '@noble/hashes': 1.8.0
+ '@scure/bip32': 1.7.0
+ '@scure/bip39': 1.6.0
+ optionalDependencies:
+ typescript: 5.8.3
+ transitivePeerDependencies:
+ - bufferutil
+ - utf-8-validate
+ - zod
+
dependencies:
'@noble/curves': 1.9.2
@@ -15355,14 +15388,14 @@ snapshots:
dependencies:
xml-name-validator: 5.0.0
dependencies:
'@tanstack/react-query': 5.84.1([email protected])
react: 18.3.1
use-sync-external-store: 1.4.0([email protected])
optionalDependencies:
typescript: 5.8.3
transitivePeerDependencies:
@@ -15569,8 +15602,8 @@ snapshots:
zod: 3.25.76
transitivePeerDependencies:
- '@azure/app-configuration'
diff --git a/apps/api/src/lib/search-index-client.ts b/apps/api/src/lib/search-index-client.ts
index bfee651cf..dd0f1da67 100644
--- a/apps/api/src/lib/search-index-client.ts
+++ b/apps/api/src/lib/search-index-client.ts
@@ -1,6 +1,6 @@
/**
* HTTP Client for Search Index Service
- *
+ *
* This client communicates with the standalone search index service
* (firecrawl-search backend) via HTTP.
*/
@@ -8,13 +8,13 @@
import { logger as _logger } from "./logger";
import type { Logger } from "winston";
-export interface SearchIndexClientConfig {
+interface SearchIndexClientConfig {
baseUrl: string;
apiSecret?: string;
timeout?: number;
}
-export interface IndexDocumentRequest {
+interface IndexDocumentRequest {
url: string;
resolvedUrl: string;
title?: string;
@@ -43,7 +43,7 @@ export interface SearchRequest {
};
}
-export interface SearchResult {
+interface SearchResult {
documentId: string;
url: string;
title: string | null;
@@ -58,7 +58,7 @@ export interface SearchResult {
lastCrawledAt: string;
}
-export interface SearchResponse {
+interface SearchResponse {
results: SearchResult[];
total: number;
query: string;
@@ -75,13 +75,13 @@ export class SearchIndexClient {
private baseUrl: string;
private apiSecret?: string;
private timeout: number;
-
+
constructor(config: SearchIndexClientConfig) {
this.baseUrl = config.baseUrl.replace(/\/$/, ""); // Remove trailing slash
this.apiSecret = config.apiSecret;
this.timeout = config.timeout || 30000;
}
-
+
/**
* Check if search index service is enabled
*/
@@ -91,7 +91,7 @@ export class SearchIndexClient {
process.env.ENABLE_SEARCH_INDEX === "true"
);
}
-
+
/**
* Make HTTP request to search service
*/
@@ -103,36 +103,36 @@ export class SearchIndexClient {
): Promise<T> {
const log = logger ?? _logger.child({ module: "search-index-client" });
const url = `${this.baseUrl}${path}`;
-
+
const headers: Record<string, string> = {
"Content-Type": "application/json",
};
-
+
if (this.apiSecret) {
headers["X-API-Secret"] = this.apiSecret;
}
-
+
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
-
+
try {
log.debug("Making request to search service", {
method,
url,
hasBody: !!body,
});
-
+
const response = await fetch(url, {
method,
headers,
body: body ? JSON.stringify(body) : undefined,
signal: controller.signal,
});
-
+
clearTimeout(timeoutId);
-
+
const data = await response.json();
-
+
if (!response.ok) {
log.error("Search service request failed", {
status: response.status,
@@ -142,16 +142,19 @@ export class SearchIndexClient {
data.error || `Search service returned ${response.status}`,
);
}
-
+
return data;
} catch (error) {
clearTimeout(timeoutId);
-
+
if (error.name === "AbortError") {
- log.error("Search service request timed out", { url, timeout: this.timeout });
+ log.error("Search service request timed out", {
+ url,
+ timeout: this.timeout,
+ });
throw new Error("Search service request timed out");
}
-
+
log.error("Search service request failed", {
error: (error as Error).message,
url,
@@ -159,7 +162,7 @@ export class SearchIndexClient {
throw error;
}
}
-
+
/**
* Index a document (async - queues for processing)
*/
@@ -168,7 +171,7 @@ export class SearchIndexClient {
logger?: Logger,
): Promise<{ success: boolean; message: string }> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
const response = await this.request<any>(
"POST",
@@ -176,12 +179,12 @@ export class SearchIndexClient {
request,
log,
);
-
+
log.info("Document queued for indexing", {
url: request.url,
success: response.success,
});
-
+
return {
success: response.success,
message: response.message || "Document queued for indexing",
@@ -191,7 +194,7 @@ export class SearchIndexClient {
error: (error as Error).message,
url: request.url,
});
-
+
// Don't throw - indexing failures shouldn't break scraping
return {
success: false,
@@ -199,7 +202,7 @@ export class SearchIndexClient {
};
}
}
-
+
/**
* Search indexed documents
*/
@@ -208,32 +211,30 @@ export class SearchIndexClient {
logger?: Logger,
): Promise<SearchResponse> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
- const response = await this.request<{ success: boolean; data: SearchResponse }>(
- "POST",
- "/api/search",
- request,
- log,
- );
-
+ const response = await this.request<{
+ success: boolean;
+ data: SearchResponse;
+ }>("POST", "/api/search", request, log);
+
if (!response.success || !response.data) {
throw new Error("Invalid response from search service");
}
-
+
log.info("Search completed", {
query: request.query,
results: response.data.results.length,
took: response.data.took,
});
-
+
return response.data;
} catch (error) {
log.error("Search request failed", {
error: (error as Error).message,
query: request.query,
});
-
+
// Return empty results on error
return {
results: [],
@@ -249,7 +250,7 @@ export class SearchIndexClient {
};
}
}
-
+
/**
* Get search index statistics
*/
@@ -258,7 +259,7 @@ export class SearchIndexClient {
queue: any;
}> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
const response = await this.request<{ success: boolean; data: any }>(
"GET",
@@ -266,30 +267,30 @@ export class SearchIndexClient {
undefined,
log,
);
-
+
if (!response.success || !response.data) {
throw new Error("Invalid response from search service");
}
-
+
return response.data;
} catch (error) {
log.error("Failed to get search stats", {
error: (error as Error).message,
});
-
+
return {
index: {},
queue: {},
};
}
}
-
+
/**
* Health check
*/
async health(logger?: Logger): Promise<boolean> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
const response = await this.request<{ success: boolean }>(
"GET",
@@ -297,7 +298,7 @@ export class SearchIndexClient {
undefined,
log,
);
-
+
return response.success;
} catch (error) {
log.warn("Search service health check failed", {
@@ -318,22 +319,22 @@ export function getSearchIndexClient(): SearchIndexClient | null {
if (!SearchIndexClient.isEnabled()) {
return null;
}
-
+
if (!searchIndexClient) {
const baseUrl = process.env.SEARCH_SERVICE_URL!;
const apiSecret = process.env.SEARCH_SERVICE_API_SECRET;
-
+
searchIndexClient = new SearchIndexClient({
baseUrl,
apiSecret,
timeout: 30000,
});
-
+
_logger.info("Search index client initialized", {
baseUrl: baseUrl.substring(0, 30) + "...",
});
}
-
+
return searchIndexClient;
}
@@ -345,11 +346,11 @@ export async function indexDocumentIfEnabled(
logger?: Logger,
): Promise<void> {
const client = getSearchIndexClient();
-
+
if (!client) {
return;
}
-
+
try {
await client.indexDocument(request, logger);
} catch (error) {
@@ -360,4 +361,3 @@ export async function indexDocumentIfEnabled(
});
}
}
-
diff --git a/apps/api/src/services/search-index-db.ts b/apps/api/src/services/search-index-db.ts
deleted file mode 100644
index 8b199f323..000000000
--- a/apps/api/src/services/search-index-db.ts
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Search Index Database Service
- *
- * Separate Supabase client for the search index database.
- * This allows independent scaling and isolation from the main index DB.
- */
-
-import { createClient, SupabaseClient } from "@supabase/supabase-js";
-import { logger as _logger } from "../lib/logger";
-import { configDotenv } from "dotenv";
-
-configDotenv();
-
-/**
- * SearchIndexSupabaseService - Manages connection to search index database
- */
-class SearchIndexSupabaseService {
- private client: SupabaseClient | null = null;
-
- constructor() {
- const supabaseUrl = process.env.SEARCH_INDEX_SUPABASE_URL;
- const supabaseServiceToken = process.env.SEARCH_INDEX_SUPABASE_SERVICE_TOKEN;
-
- // Only initialize if both URL and token are provided
- if (!supabaseUrl || !supabaseServiceToken) {
- _logger.warn("Search index database not configured. Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN to enable.");
- this.client = null;
- } else {
- this.client = createClient(supabaseUrl, supabaseServiceToken);
- _logger.info("Search index database client initialized", {
- url: supabaseUrl.substring(0, 30) + "...",
- });
- }
- }
-
- /**
- * Get the Supabase client for search index
- */
- getClient(): SupabaseClient | null {
- return this.client;
- }
-
- /**
- * Check if search index is enabled
- */
- isEnabled(): boolean {
- return this.client !== null;
- }
-}
-
-const searchIndexService = new SearchIndexSupabaseService();
-
-/**
- * Proxy to provide clean error messages when search index is not configured
- */
-export const search_index_supabase_service: SupabaseClient = new Proxy(
- searchIndexService,
- {
- get: function (target, prop, receiver) {
- const client = target.getClient();
-
- // If client is not initialized, throw immediately to prevent nested access errors
- if (client === null) {
- throw new Error(
- "Search index database is not configured. " +
- "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN environment variables."
- );
- }
-
- // Direct access to service properties
- if (prop in target) {
- return Reflect.get(target, prop, receiver);
- }
-
- // Delegate to Supabase client
- return Reflect.get(client, prop, receiver);
- },
- }
-) as unknown as SupabaseClient;
-
-/**
- * Check if search index database is enabled
- */
-export function isSearchIndexEnabled(): boolean {
- return (
- process.env.ENABLE_SEARCH_INDEX === "true" &&
- searchIndexService.isEnabled()
- );
-}
-
-/**
- * Get search index database client (throws if not configured)
- */
-export function getSearchIndexClient(): SupabaseClient {
- const client = searchIndexService.getClient();
- if (!client) {
- throw new Error(
- "Search index database is not configured. " +
- "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN."
- );
- }
- return client;
-}
-