Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolasee0987f07999
diff --git a/apps/api/src/controllers/v2/f-search.ts b/apps/api/src/controllers/v2/f-search.tsnew file mode 100644index 000000000..482390dbf--- /dev/null+++ b/apps/api/src/controllers/v2/f-search.ts@@ -0,0 +1,249 @@+import { Request, Response } from "express";+import { z } from "zod";+import {+ search_index_supabase_service,+ isSearchIndexEnabled,+} from "../../services/search-index-db";+import {+ search,+ searchChunks,+ getSearchStats,+ type SearchQuery,+ type SearchFilters,+} from "../../lib/search-index";+import { logger as _logger } from "../../lib/logger";+import { CostTracking } from "../../lib/cost-tracking";++// Validation schemas+const searchRequestSchema = z.object({+ query: z.string().min(1).max(500),+ limit: z.number().int().min(1).max(100).optional().default(50),+ offset: z.number().int().min(0).optional().default(0),+ mode: z.enum(["hybrid", "keyword", "semantic"]).optional().default("hybrid"),+ filters: z+ .object({+ domain: z.string().optional(),+ country: z.string().length(2).optional(),+ isMobile: z.boolean().optional(),+ minFreshness: z.number().min(0).max(1).optional(),+ language: z.string().optional(),+ })+ .optional()+ .default({}),+});++const searchChunksRequestSchema = z.object({+ query: z.string().min(1).max(500),+ limit: z.number().int().min(1).max(50).optional().default(20),+ filters: z+ .object({+ domain: z.string().optional(),+ })+ .optional()+ .default({}),+});++export async function realtimeSearchController(+ req: Request,+ res: Response,+): Promise<void> {+ const logger = _logger.child({+ module: "realtime-search-controller",+ method: "POST /admin/search",+ teamId: (req as any).auth?.team_id,+ });++ try {+ // Validate request+ const validationResult = searchRequestSchema.safeParse(req.body);++ if (!validationResult.success) {+ res.status(400).json({+ success: false,+ error: "Invalid request parameters",+ details: validationResult.error.errors,+ });+ return;+ }++ const { query, limit, offset, mode, filters } = validationResult.data;++ logger.info("Search request", {+ query,+ mode,+ limit,+ filters,+ });++ // Check if search index is enabled+ if (!isSearchIndexEnabled()) {+ res.status(503).json({+ success: false,+ error: "Search index is not configured",+ });+ return;+ }++ // Perform search+ const searchQuery: SearchQuery = {+ query,+ limit,+ offset,+ mode,+ filters: filters as SearchFilters,+ };++ const result = await search(+ search_index_supabase_service,+ searchQuery,+ logger,+ );++ // Track cost (if applicable)+ const costTracking = new CostTracking();+ // Add search cost tracking here if needed++ res.status(200).json({+ success: true,+ data: {+ results: result.results,+ total: result.total,+ query: result.query,+ mode: result.mode,+ took: result.took,+ pagination: {+ limit,+ offset,+ hasMore: result.total > offset + limit,+ },+ },+ costTracking: costTracking.toJSON(),+ });+ } catch (error) {+ logger.error("Search request failed", {+ error: (error as Error).message,+ });++ res.status(500).json({+ success: false,+ error: "Internal server error",+ message: (error as Error).message,+ });+ }+}++// /**+// * POST /admin/search/chunks+// * Chunk-level search for precise snippets+// */+// export async function searchChunksController(+// req: Request,+// res: Response,+// ): Promise<void> {+// const logger = _logger.child({+// module: "search-chunks-controller",+// method: "POST /admin/search/chunks",+// teamId: (req as any).auth?.team_id,+// });++// try {+// // Validate request+// const validationResult = searchChunksRequestSchema.safeParse(req.body);++// if (!validationResult.success) {+// res.status(400).json({+// success: false,+// error: "Invalid request parameters",+// details: validationResult.error.errors,+// });+// return;+// }++// const { query, limit, filters } = validationResult.data;++// logger.info("Chunk search request", {+// query,+// limit,+// filters,+// });++// // Check if search index is enabled+// if (!isSearchIndexEnabled()) {+// res.status(503).json({+// success: false,+// error: "Search index is not configured",+// });+// return;+// }++// // Perform chunk search+// const results = await searchChunks(+// search_index_supabase_service,+// query,+// limit,+// filters as SearchFilters,+// logger,+// );++// res.status(200).json({+// success: true,+// data: {+// chunks: results,+// total: results.length,+// query,+// },+// });+// } catch (error) {+// logger.error("Chunk search request failed", {+// error: (error as Error).message,+// });++// res.status(500).json({+// success: false,+// error: "Internal server error",+// message: (error as Error).message,+// });+// }+// }++// /**+// * GET /admin/search/stats+// * Get search index statistics+// */+// export async function searchStatsController(+// req: Request,+// res: Response,+// ): Promise<void> {+// const logger = _logger.child({+// module: "search-stats-controller",+// method: "GET /admin/search/stats",+// });++// try {+// // Check if search index is enabled+// if (!isSearchIndexEnabled()) {+// res.status(503).json({+// success: false,+// error: "Search index is not configured",+// });+// return;+// }++// const stats = await getSearchStats(search_index_supabase_service);++// res.status(200).json({+// success: true,+// data: stats,+// });+// } catch (error) {+// logger.error("Failed to get search stats", {+// error: (error as Error).message,+// });++// res.status(500).json({+// success: false,+// error: "Internal server error",+// message: (error as Error).message,+// });+// }+// }diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex 9fb1eb67f..fb3a650a6 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -1,6 +1,6 @@import express from "express";import { redisHealthController } from "../controllers/v0/admin/redis-health";-import { wrap } from "./shared";+import { authMiddleware, blocklistMiddleware, wrap } from "./shared";import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";import { cclogController } from "../controllers/v0/admin/cclog";@@ -12,6 +12,7 @@ import {nuqMetricsController,} from "../controllers/v0/admin/metrics";import { crawlCheckController } from "../controllers/v0/admin/crawl-check";+import { realtimeSearchController } from "../controllers/v2/f-search";export const adminRouter = express.Router();@@ -64,3 +65,9 @@ adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/crawl-check`,wrap(crawlCheckController),);+++adminRouter.post(+ `/admin/${process.env.BULL_AUTH_KEY}/fsearch`,+ wrap(realtimeSearchController),+);\ No newline at end of file