Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawl-errors): take advantage of TransportableError
Gergő Móricz588fe390a3b5
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex 307efe4b0..45084c741 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -14,6 +14,8 @@ import { configDotenv } from "dotenv";import { Job } from "bullmq";import { supabase_rr_service } from "../../services/supabase";import { logger } from "../../lib/logger";+import { deserializeTransportableError } from "../../lib/error-serde";+import { TransportableError } from "../../lib/error";configDotenv();export async function getJob(id: string) {@@ -57,15 +59,23 @@ export async function crawlErrorsController(}res.status(200).json({- errors: (await getJobs(failedJobIDs)).map((x) => ({- id: x.id,- timestamp:- x.finishedOn !== undefined- ? new Date(x.finishedOn).toISOString()- : undefined,- url: x.data.url,- error: x.failedReason,- })),+ errors: (await getJobs(failedJobIDs)).map((x) => {+ const error = deserializeTransportableError(x.failedReason) as TransportableError | null;+ return {+ id: x.id,+ timestamp:+ x.finishedOn !== undefined+ ? new Date(x.finishedOn).toISOString()+ : undefined,+ url: x.data.url,+ ...(error ? {+ code: error.code,+ error: error.message,+ } : {+ error: x.failedReason,+ }),+ };+ }),robotsBlocked: await redisEvictConnection.smembers("crawl:" + req.params.jobId + ":robots_blocked",),@@ -117,12 +127,23 @@ export async function crawlErrorsController(}res.status(200).json({- errors: (failedJobs || []).map((job) => ({- id: job.job_id,- timestamp: new Date(job.date_added).toISOString(),- url: job.page_options?.url || job.page_options?.urls?.[0] || "Unknown URL",- error: job.message || "Unknown error",- })),+ errors: (failedJobs || []).map((job) => {+ const error = deserializeTransportableError(job.message) as TransportableError | null;+ return {+ id: job.job_id,+ timestamp:+ job.finishedOn !== undefined+ ? new Date(job.finishedOn).toISOString()+ : undefined,+ url: job.url,+ ...(error ? {+ code: error.code,+ error: error.message,+ } : {+ error: job.message,+ }),+ };+ }),robotsBlocked: await redisEvictConnection.smembers("crawl:" + req.params.jobId + ":robots_blocked",),diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsindex 307efe4b0..45084c741 100644--- a/apps/api/src/controllers/v2/crawl-errors.ts+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -14,6 +14,8 @@ import { configDotenv } from "dotenv";import { Job } from "bullmq";import { supabase_rr_service } from "../../services/supabase";import { logger } from "../../lib/logger";+import { deserializeTransportableError } from "../../lib/error-serde";+import { TransportableError } from "../../lib/error";configDotenv();export async function getJob(id: string) {@@ -57,15 +59,23 @@ export async function crawlErrorsController(}res.status(200).json({- errors: (await getJobs(failedJobIDs)).map((x) => ({- id: x.id,- timestamp:- x.finishedOn !== undefined- ? new Date(x.finishedOn).toISOString()- : undefined,- url: x.data.url,- error: x.failedReason,- })),+ errors: (await getJobs(failedJobIDs)).map((x) => {+ const error = deserializeTransportableError(x.failedReason) as TransportableError | null;+ return {+ id: x.id,+ timestamp:+ x.finishedOn !== undefined+ ? new Date(x.finishedOn).toISOString()+ : undefined,+ url: x.data.url,+ ...(error ? {+ code: error.code,+ error: error.message,+ } : {+ error: x.failedReason,+ }),+ };+ }),robotsBlocked: await redisEvictConnection.smembers("crawl:" + req.params.jobId + ":robots_blocked",),@@ -117,12 +127,23 @@ export async function crawlErrorsController(}res.status(200).json({- errors: (failedJobs || []).map((job) => ({- id: job.job_id,- timestamp: new Date(job.date_added).toISOString(),- url: job.page_options?.url || job.page_options?.urls?.[0] || "Unknown URL",- error: job.message || "Unknown error",- })),+ errors: (failedJobs || []).map((job) => {+ const error = deserializeTransportableError(job.message) as TransportableError | null;+ return {+ id: job.job_id,+ timestamp:+ job.finishedOn !== undefined+ ? new Date(job.finishedOn).toISOString()+ : undefined,+ url: job.url,+ ...(error ? {+ code: error.code,+ error: error.message,+ } : {+ error: job.message,+ }),+ };+ }),robotsBlocked: await redisEvictConnection.smembers("crawl:" + req.params.jobId + ":robots_blocked",),diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 9068eda7a..64f0e7552 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -867,6 +867,7 @@ export type CrawlErrorsResponse =id: string;timestamp?: string;url: string;+ code?: ErrorCodes;error: string;}[];robotsBlocked: string[];