A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
fix(workers): denoise the log messages coming from the feedWorker
Mohamed Bassemd155fa0eece3
diff --git a/apps/workers/workers/feedWorker.ts b/apps/workers/workers/feedWorker.tsindex 340eeff4..16b0fcd5 100644--- a/apps/workers/workers/feedWorker.ts+++ b/apps/workers/workers/feedWorker.ts@@ -17,6 +17,8 @@ import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";import { parseFeedItems } from "./utils/feedParser";+type FeedRunResult = "success" | "failure";+/*** Deterministically maps a feed ID to a minute offset within the hour (0-59).* This ensures feeds are spread evenly across the hour based on their ID.@@ -111,20 +113,25 @@ export const FeedRefreshingWorker = cron.schedule(export class FeedWorker {static async build() {logger.info("Starting feed worker ...");- const worker = (await getQueueClient())!.createRunner<ZFeedRequestSchema>(+ const worker = (await getQueueClient())!.createRunner<+ ZFeedRequestSchema,+ FeedRunResult+ >(FeedQueue,{run: withWorkerTracing("feedWorker.run",withWorkerEventLog("feedWorker.run", run),),- onComplete: async (job) => {+ onComplete: async (job, result) => {workerStatsCounter.labels("feed", "completed").inc();const jobId = job.id;- logger.info(`[feed][${jobId}] Completed successfully`);+ logger.info(+ `[feed][${jobId}] Completed with fetch status: ${result}`,+ );await db.update(rssFeedsTable)- .set({ lastFetchedStatus: "success", lastFetchedAt: new Date() })+ .set({ lastFetchedStatus: result, lastFetchedAt: new Date() }).where(eq(rssFeedsTable.id, job.data?.feedId));},onError: async (job) => {@@ -155,7 +162,9 @@ export class FeedWorker {}}-async function run(req: DequeuedJob<ZFeedRequestSchema>) {+async function run(+ req: DequeuedJob<ZFeedRequestSchema>,+): Promise<FeedRunResult> {const jobId = req.id;addLogFields<"feedWorker.run">({ "feed.id": req.data.feedId });const feed = await db.query.rssFeedsTable.findFirst({@@ -179,7 +188,7 @@ async function run(req: DequeuedJob<ZFeedRequestSchema>) {`[feed][${jobId}] User ${feed.userId} doesn't have enough quota to create bookmarks. Skipping feed fetching.`,);addLogFields<"feedWorker.run">({ "feed.skipped_quota": true });- return;+ return "success";}}@@ -187,33 +196,62 @@ async function run(req: DequeuedJob<ZFeedRequestSchema>) {`[feed][${jobId}] Starting fetching feed "${feed.name}" (${feed.id}) ...`,);- const response = await fetchWithProxy(feed.url, {- signal: AbortSignal.timeout(5000),- headers: {- "User-Agent":- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",- Accept: "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8",- },- });+ let response;+ try {+ response = await fetchWithProxy(feed.url, {+ signal: AbortSignal.timeout(5000),+ headers: {+ "User-Agent":+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",+ Accept: "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8",+ },+ });+ } catch (error) {+ logger.warn(+ `[feed][${jobId}] Failed to fetch feed "${feed.name}" (${feed.id}): ${getErrorMessage(error)}. Skipping until the next scheduled fetch.`,+ );+ return "failure";+ }+addLogFields<"feedWorker.run">({ "feed.status_code": response.status });if (response.status !== 200) {- throw new Error(- `[feed][${jobId}] Feed "${feed.name}" (${feed.id}) returned a non-success status: ${response.status}.`,+ logger.warn(+ `[feed][${jobId}] Feed "${feed.name}" (${feed.id}) returned a non-success status: ${response.status}. Skipping until the next scheduled fetch.`,);+ return "failure";}const contentType = response.headers.get("content-type");if (!contentType || !contentType.includes("xml")) {- throw new Error(- `[feed][${jobId}] Feed "${feed.name}" (${feed.id}) is not a valid RSS feed`,+ logger.warn(+ `[feed][${jobId}] Feed "${feed.name}" (${feed.id}) is not a valid RSS feed. Skipping until the next scheduled fetch.`,+ );+ return "failure";+ }++ let xmlData;+ try {+ xmlData = await response.text();+ } catch (error) {+ logger.warn(+ `[feed][${jobId}] Failed to read feed "${feed.name}" (${feed.id}): ${getErrorMessage(error)}. Skipping until the next scheduled fetch.`,);+ return "failure";}- const xmlData = await response.text();logger.info(`[feed][${jobId}] Successfully fetched feed "${feed.name}" (${feed.id}) ...`,);- const feedItems = await parseFeedItems(xmlData);+ let feedItems;+ try {+ feedItems = await parseFeedItems(xmlData);+ } catch (error) {+ logger.warn(+ `[feed][${jobId}] Failed to parse feed "${feed.name}" (${feed.id}): ${getErrorMessage(error)}. Skipping until the next scheduled fetch.`,+ );+ return "failure";+ }+addLogFields<"feedWorker.run">({ "feed.items_found": feedItems.length });await db.update(rssFeedsTable)@@ -226,7 +264,7 @@ async function run(req: DequeuedJob<ZFeedRequestSchema>) {if (feedItems.length === 0) {logger.info(`[feed][${jobId}] No entries found.`);- return;+ return "success";}const exitingEntries = await db.query.rssFeedImportsTable.findMany({@@ -251,7 +289,7 @@ async function run(req: DequeuedJob<ZFeedRequestSchema>) {logger.info(`[feed][${jobId}] No new entries found in feed "${feed.name}" (${feed.id}).`,);- return;+ return "success";}logger.info(@@ -338,5 +376,5 @@ async function run(req: DequeuedJob<ZFeedRequestSchema>) {`[feed][${jobId}] Imported ${newEntries.length} feed entries from feed "${feed.name}" (${feed.id}); created ${createdBookmarkCount} bookmarks${bookmarkCreationFailures.length > 0 ? `, failed to create ${bookmarkCreationFailures.length}` : ""}.`,);- return Promise.resolve();+ return "success";}