Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
push
Gergő Móricz17d0ed061ea6
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex cf905207e..794390389 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -7,7 +7,6 @@ import cors from "cors";import { getScrapeQueue } from "./services/queue-service";import { v0Router } from "./routes/v0";import { initSDK } from "@hyperdx/node-opentelemetry";-import cluster from "cluster";import os from "os";import { Logger } from "./lib/logger";import { adminRouter } from "./routes/admin";@@ -37,198 +36,179 @@ const cacheable = new CacheableLookup({cacheable.install(http.globalAgent);cacheable.install(https.globalAgent)-if (cluster.isMaster) {- Logger.info(`Master ${process.pid} is running`);+const ws = expressWs(express());+const app = ws.app;- // Fork workers.- for (let i = 0; i < numCPUs; i++) {- cluster.fork();- }+global.isProduction = process.env.IS_PRODUCTION === "true";- cluster.on("exit", (worker, code, signal) => {- if (code !== null) {- Logger.info(`Worker ${worker.process.pid} exited`);- Logger.info("Starting a new worker");- cluster.fork();- }- });-} else {- const ws = expressWs(express());- const app = ws.app;+app.use(bodyParser.urlencoded({ extended: true }));+app.use(bodyParser.json({ limit: "10mb" }));- global.isProduction = process.env.IS_PRODUCTION === "true";+app.use(cors()); // Add this line to enable CORS- app.use(bodyParser.urlencoded({ extended: true }));- app.use(bodyParser.json({ limit: "10mb" }));+const serverAdapter = new ExpressAdapter();+serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);- app.use(cors()); // Add this line to enable CORS+const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({+ queues: [new BullAdapter(getScrapeQueue())],+ serverAdapter: serverAdapter,+});- const serverAdapter = new ExpressAdapter();- serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);+app.use(+ `/admin/${process.env.BULL_AUTH_KEY}/queues`,+ serverAdapter.getRouter()+);- const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({- queues: [new BullAdapter(getScrapeQueue())],- serverAdapter: serverAdapter,- });+app.get("/", (req, res) => {+ res.send("SCRAPERS-JS: Hello, world! Fly.io");+});- app.use(- `/admin/${process.env.BULL_AUTH_KEY}/queues`,- serverAdapter.getRouter()- );+//write a simple test function+app.get("/test", async (req, res) => {+ res.send("Hello, world!");+});- app.get("/", (req, res) => {- res.send("SCRAPERS-JS: Hello, world! Fly.io");- });+// register router+app.use(v0Router);+app.use("/v1", v1Router);+app.use(adminRouter);- //write a simple test function- app.get("/test", async (req, res) => {- res.send("Hello, world!");- });+const DEFAULT_PORT = process.env.PORT ?? 3002;+const HOST = process.env.HOST ?? "localhost";- // register router- app.use(v0Router);- app.use("/v1", v1Router);- app.use(adminRouter);+// HyperDX OpenTelemetry+if (process.env.ENV === "production") {+ initSDK({ consoleCapture: true, additionalInstrumentations: [] });+}- const DEFAULT_PORT = process.env.PORT ?? 3002;- const HOST = process.env.HOST ?? "localhost";+function startServer(port = DEFAULT_PORT) {+ const server = app.listen(Number(port), HOST, () => {+ Logger.info(`Worker ${process.pid} listening on port ${port}`);+ Logger.info(+ `For the Queue UI, open: http://${HOST}:${port}/admin/${process.env.BULL_AUTH_KEY}/queues`+ );+ });+ return server;+}- // HyperDX OpenTelemetry- if (process.env.ENV === "production") {- initSDK({ consoleCapture: true, additionalInstrumentations: [] });- }+if (require.main === module) {+ startServer();+}- function startServer(port = DEFAULT_PORT) {- const server = app.listen(Number(port), HOST, () => {- Logger.info(`Worker ${process.pid} listening on port ${port}`);- Logger.info(- `For the Queue UI, open: http://${HOST}:${port}/admin/${process.env.BULL_AUTH_KEY}/queues`- );+app.get(`/serverHealthCheck`, async (req, res) => {+ try {+ const scrapeQueue = getScrapeQueue();+ const [waitingJobs] = await Promise.all([+ scrapeQueue.getWaitingCount(),+ ]);++ const noWaitingJobs = waitingJobs === 0;+ // 200 if no active jobs, 503 if there are active jobs+ return res.status(noWaitingJobs ? 200 : 500).json({+ waitingJobs,});- return server;+ } catch (error) {+ Sentry.captureException(error);+ Logger.error(error);+ return res.status(500).json({ error: error.message });}+});- if (require.main === module) {- startServer();- }+app.get("/serverHealthCheck/notify", async (req, res) => {+ if (process.env.SLACK_WEBHOOK_URL) {+ const treshold = 1; // The treshold value for the active jobs+ const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds- app.get(`/serverHealthCheck`, async (req, res) => {- try {+ const getWaitingJobsCount = async () => {const scrapeQueue = getScrapeQueue();- const [waitingJobs] = await Promise.all([+ const [waitingJobsCount] = await Promise.all([scrapeQueue.getWaitingCount(),]);- const noWaitingJobs = waitingJobs === 0;- // 200 if no active jobs, 503 if there are active jobs- return res.status(noWaitingJobs ? 200 : 500).json({- waitingJobs,- });- } catch (error) {- Sentry.captureException(error);- Logger.error(error);- return res.status(500).json({ error: error.message });- }- });-- app.get("/serverHealthCheck/notify", async (req, res) => {- if (process.env.SLACK_WEBHOOK_URL) {- const treshold = 1; // The treshold value for the active jobs- const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds-- const getWaitingJobsCount = async () => {- const scrapeQueue = getScrapeQueue();- const [waitingJobsCount] = await Promise.all([- scrapeQueue.getWaitingCount(),- ]);-- return waitingJobsCount;- };-- res.status(200).json({ message: "Check initiated" });-- const checkWaitingJobs = async () => {- try {- let waitingJobsCount = await getWaitingJobsCount();- if (waitingJobsCount >= treshold) {- setTimeout(async () => {- // Re-check the waiting jobs count after the timeout- waitingJobsCount = await getWaitingJobsCount();- if (waitingJobsCount >= treshold) {- const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL;- const message = {- text: `⚠️ Warning: The number of active jobs (${waitingJobsCount}) has exceeded the threshold (${treshold}) for more than ${- timeout / 60000- } minute(s).`,- };-- const response = await fetch(slackWebhookUrl, {- method: "POST",- headers: {- "Content-Type": "application/json",- },- body: JSON.stringify(message),- });-- if (!response.ok) {- Logger.error("Failed to send Slack notification");- }+ return waitingJobsCount;+ };++ res.status(200).json({ message: "Check initiated" });++ const checkWaitingJobs = async () => {+ try {+ let waitingJobsCount = await getWaitingJobsCount();+ if (waitingJobsCount >= treshold) {+ setTimeout(async () => {+ // Re-check the waiting jobs count after the timeout+ waitingJobsCount = await getWaitingJobsCount();+ if (waitingJobsCount >= treshold) {+ const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL;+ const message = {+ text: `⚠️ Warning: The number of active jobs (${waitingJobsCount}) has exceeded the threshold (${treshold}) for more than ${+ timeout / 60000+ } minute(s).`,+ };++ const response = await fetch(slackWebhookUrl, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ },+ body: JSON.stringify(message),+ });++ if (!response.ok) {+ Logger.error("Failed to send Slack notification");}- }, timeout);- }- } catch (error) {- Sentry.captureException(error);- Logger.debug(error);+ }+ }, timeout);}- };+ } catch (error) {+ Sentry.captureException(error);+ Logger.debug(error);+ }+ };- checkWaitingJobs();- }- });+ checkWaitingJobs();+ }+});- app.get("/is-production", (req, res) => {- res.send({ isProduction: global.isProduction });- });+app.get("/is-production", (req, res) => {+ res.send({ isProduction: global.isProduction });+});- app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response<ErrorResponse>, next: NextFunction) => {- if (err instanceof ZodError) {- if (Array.isArray(err.errors) && err.errors.find(x => x.message === "URL uses unsupported protocol")) {- Logger.warn("Unsupported protocol error: " + JSON.stringify(req.body));- }+app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response<ErrorResponse>, next: NextFunction) => {+ if (err instanceof ZodError) {+ if (Array.isArray(err.errors) && err.errors.find(x => x.message === "URL uses unsupported protocol")) {+ Logger.warn("Unsupported protocol error: " + JSON.stringify(req.body));+ }- res.status(400).json({ success: false, error: "Bad Request", details: err.errors });- } else {- next(err);- }- });+ res.status(400).json({ success: false, error: "Bad Request", details: err.errors });+ } else {+ next(err);+ }+});- Sentry.setupExpressErrorHandler(app);+Sentry.setupExpressErrorHandler(app);- app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: ResponseWithSentry<ErrorResponse>, next: NextFunction) => {- if (err instanceof SyntaxError && 'status' in err && err.status === 400 && 'body' in err) {- return res.status(400).json({ success: false, error: 'Bad request, malformed JSON' });- }+app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: ResponseWithSentry<ErrorResponse>, next: NextFunction) => {+ if (err instanceof SyntaxError && 'status' in err && err.status === 400 && 'body' in err) {+ return res.status(400).json({ success: false, error: 'Bad request, malformed JSON' });+ }- const id = res.sentry ?? uuidv4();- let verbose = JSON.stringify(err);- if (verbose === "{}") {- if (err instanceof Error) {- verbose = JSON.stringify({- message: err.message,- name: err.name,- stack: err.stack,- });- }+ const id = res.sentry ?? uuidv4();+ let verbose = JSON.stringify(err);+ if (verbose === "{}") {+ if (err instanceof Error) {+ verbose = JSON.stringify({+ message: err.message,+ name: err.name,+ stack: err.stack,+ });}+ }- Logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose);- res.status(500).json({ success: false, error: "An unexpected error occurred. Please contact [email protected] for help. Your exception ID is " + id });- });-- Logger.info(`Worker ${process.pid} started`);-}-+ Logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose);+ res.status(500).json({ success: false, error: "An unexpected error occurred. Please contact [email protected] for help. Your exception ID is " + id });+});+Logger.info(`Worker ${process.pid} started`);// const sq = getScrapeQueue();