Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: wip
Nicolas86e34d7c6cf6
diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.tsindex 8ab2d11b3..feba30ec3 100644--- a/apps/api/src/controllers/v1/extract-status.ts+++ b/apps/api/src/controllers/v1/extract-status.ts@@ -1,53 +1,21 @@import { Response } from "express";-import {- supabaseGetJobByIdOnlyData,- supabaseGetJobsById,-} from "../../lib/supabase-jobs";-import { scrapeStatusRateLimiter } from "../../services/rate-limiter";+import { supabaseGetJobsById } from "../../lib/supabase-jobs";import { RequestWithAuth } from "./types";export async function extractStatusController(req: RequestWithAuth<{ jobId: string }, any, any>,res: Response,) {- try {- const rateLimiter = scrapeStatusRateLimiter;- const incomingIP = (req.headers["x-forwarded-for"] ||- req.socket.remoteAddress) as string;- const iptoken = incomingIP;- await rateLimiter.consume(iptoken);-- const job = await supabaseGetJobByIdOnlyData(req.params.jobId);- if (!job || job.team_id !== req.auth.team_id) {- return res.status(403).json({- success: false,- error: "You are not allowed to access this resource.",- });- }-- const jobData = await supabaseGetJobsById([req.params.jobId]);- if (!jobData || jobData.length === 0) {- return res.status(404).json({- success: false,- error: "Job not found",- });- }-- return res.status(200).json({- success: true,- data: jobData[0].docs,+ const jobData = await supabaseGetJobsById([req.params.jobId]);+ if (!jobData || jobData.length === 0) {+ return res.status(404).json({+ success: false,+ error: "Job not found",});- } catch (error) {- if (error instanceof Error && error.message == "Too Many Requests") {- return res.status(429).json({- success: false,- error: "Rate limit exceeded. Please try again later.",- });- } else {- return res.status(500).json({- success: false,- error: "An unexpected error occurred.",- });- }}++ return res.status(200).json({+ success: true,+ data: jobData[0].docs,+ });}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 422a8160d..a8a2e408a 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -351,7 +351,7 @@ const processExtractJobInternal = async (token: string, job: Job & { id: string// Move job to failed state in Redisawait job.moveToFailed(error, token, false);- throw error;+ // throw error;} finally {clearInterval(extendLockInterval);}