A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
fix(workers): handle TWZ RSS category objects (#2589)
T.J. Tarazevits645f2f0b2aaa
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 13500f20..199fe49b 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -64,6 +64,10 @@ jobs:working-directory: packages/trpcrun: pnpm test+ - name: Workers Tests+ working-directory: apps/workers+ run: pnpm test+- name: E2E Testsworking-directory: packages/e2e_testsrun: pnpm testdiff --git a/apps/workers/package.json b/apps/workers/package.jsonindex 5c7277f9..7fbcd06f 100644--- a/apps/workers/package.json+++ b/apps/workers/package.json@@ -62,7 +62,8 @@"@types/archiver": "^7.0.0","@types/jsdom": "^21.1.6","@types/node-cron": "^3.0.11",- "tsdown": "^0.12.9"+ "tsdown": "^0.12.9",+ "vitest": "^3.2.4"},"scripts": {"start": "tsx watch index.ts",@@ -73,6 +74,8 @@"lint:fix": "oxlint . --fix","format": "oxfmt --check .","format:fix": "oxfmt .",+ "test": "vitest run",+ "test:watch": "vitest","typecheck": "tsc --noEmit"}}diff --git a/apps/workers/workers/feedWorker.ts b/apps/workers/workers/feedWorker.tsindex eed7ccb1..0adba9f7 100644--- a/apps/workers/workers/feedWorker.ts+++ b/apps/workers/workers/feedWorker.ts@@ -2,10 +2,8 @@ import { and, eq, inArray } from "drizzle-orm";import { workerStatsCounter } from "metrics";import { fetchWithProxy } from "network";import cron from "node-cron";-import Parser from "rss-parser";import { buildImpersonatingTRPCClient } from "trpc";import { withWorkerTracing } from "workerTracing";-import { z } from "zod";import type { ZFeedRequestSchema } from "@karakeep/shared-server";import { db } from "@karakeep/db";@@ -15,6 +13,8 @@ import logger from "@karakeep/shared/logger";import { DequeuedJob, getQueueClient } from "@karakeep/shared/queueing";import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";+import { parseFeedItems } from "./utils/feedParser";+/*** Deterministically maps a feed ID to a minute offset within the hour (0-59).* This ensures feeds are spread evenly across the hour based on their ID.@@ -178,26 +178,7 @@ async function run(req: DequeuedJob<ZFeedRequestSchema>) {`[feed][${jobId}] Successfully fetched feed "${feed.name}" (${feed.id}) ...`,);- const parser = new Parser({- customFields: {- item: ["id"],- },- });- const unparseFeedData = await parser.parseString(xmlData);-- // Apparently, we can't trust the output of the xml parser. So let's do our own type- // validation.- const feedItemsSchema = z.object({- id: z.coerce.string(),- link: z.string().optional(),- guid: z.string().optional(),- title: z.string().optional(),- categories: z.array(z.string()).optional(),- });-- const feedItems = unparseFeedData.items- .map((i) => feedItemsSchema.safeParse(i))- .flatMap((i) => (i.success ? [i.data] : []));+ const feedItems = await parseFeedItems(xmlData);logger.info(`[feed][${jobId}] Found ${feedItems.length} entries in feed "${feed.name}" (${feed.id}) ...`,@@ -208,11 +189,6 @@ async function run(req: DequeuedJob<ZFeedRequestSchema>) {return;}- // For feeds that don't have guids, use the link as the id- feedItems.forEach((item) => {- item.guid = item.guid ?? item.id ?? item.link;- });-const exitingEntries = await db.query.rssFeedImportsTable.findMany({where: and(eq(rssFeedImportsTable.rssFeedId, feed.id),diff --git a/apps/workers/workers/utils/__fixtures__/twz-feed.xml b/apps/workers/workers/utils/__fixtures__/twz-feed.xmlnew file mode 100644index 00000000..4e823709--- /dev/null+++ b/apps/workers/workers/utils/__fixtures__/twz-feed.xml@@ -0,0 +1,23 @@+<?xml version="1.0" encoding="UTF-8"?>+<rss+ xmlns:content="http://purl.org/rss/1.0/modules/content/"+ xmlns:dc="http://purl.org/dc/elements/1.1/"+ version="2.0"+>+ <channel>+ <title>The War Zone</title>+ <link>https://www.twz.com</link>+ <description><![CDATA[A strong offense for the world of defense.]]></description>+ <item>+ <title><![CDATA[Test TWZ article]]></title>+ <description><![CDATA[<p>Test description.</p>]]></description>+ <link>https://www.twz.com/sea/test-article</link>+ <guid isPermaLink="false">https://www.twz.com/?p=12345</guid>+ <pubDate>Tue, 14 Jan 2026 10:00:00 -0500</pubDate>+ <category domain="https://www.twz.com/category/sea">Sea</category>+ <category domain="https://www.twz.com/category/news-features">News & Features</category>+ <content:encoded><![CDATA[<p>Test content.</p>]]></content:encoded>+ <dc:creator><![CDATA[Test Author]]></dc:creator>+ </item>+ </channel>+</rss>diff --git a/apps/workers/workers/utils/feedParser.test.ts b/apps/workers/workers/utils/feedParser.test.tsnew file mode 100644index 00000000..e36761b7--- /dev/null+++ b/apps/workers/workers/utils/feedParser.test.ts@@ -0,0 +1,50 @@+import { readFile } from "node:fs/promises";++import { describe, expect, test } from "vitest";++import { parseFeedItems } from "./feedParser";++describe("parseFeedItems", () => {+ test("parses TWZ-style RSS items without dropping them", async () => {+ const xmlData = await readFile(+ new URL("./__fixtures__/twz-feed.xml", import.meta.url),+ "utf8",+ );++ const items = await parseFeedItems(xmlData);++ expect(items).toHaveLength(1);+ expect(items[0]).toMatchObject({+ guid: "https://www.twz.com/?p=12345",+ link: "https://www.twz.com/sea/test-article",+ title: "Test TWZ article",+ categories: ["Sea", "News & Features"],+ });+ });++ test("falls back to guid when feeds do not provide an item id", async () => {+ const items = await parseFeedItems(`+ <?xml version="1.0" encoding="UTF-8"?>+ <rss version="2.0">+ <channel>+ <title>Test Feed</title>+ <link>https://example.com</link>+ <description>Test</description>+ <item>+ <guid isPermaLink="false">guid-1</guid>+ <link>https://example.com/post-1</link>+ <title>Post 1</title>+ </item>+ </channel>+ </rss>+ `);++ expect(items).toHaveLength(1);+ expect(items[0]).toMatchObject({+ guid: "guid-1",+ link: "https://example.com/post-1",+ title: "Post 1",+ });+ expect(items[0].id).toBeUndefined();+ });+});diff --git a/apps/workers/workers/utils/feedParser.ts b/apps/workers/workers/utils/feedParser.tsnew file mode 100644index 00000000..4b485999--- /dev/null+++ b/apps/workers/workers/utils/feedParser.ts@@ -0,0 +1,42 @@+import Parser from "rss-parser";+import { z } from "zod";++const parser = new Parser({+ customFields: {+ item: ["id"],+ },+});++const categorySchema = z+ .union([z.string(), z.object({ _: z.string() })])+ .transform((c) => (typeof c === "string" ? c : c._));++const optionalStringSchema = z.preprocess(+ (value) => (typeof value === "string" ? value : undefined),+ z.string().optional(),+);++const feedItemSchema = z+ .object({+ id: optionalStringSchema,+ link: z.string().optional(),+ guid: z.string().optional(),+ title: z.string().optional(),+ categories: z.array(categorySchema).optional(),+ })+ .transform((item) => ({+ ...item,+ guid: item.guid ?? item.id ?? item.link,+ }));++export type ParsedFeedItem = z.infer<typeof feedItemSchema>;++export async function parseFeedItems(+ xmlData: string,+): Promise<ParsedFeedItem[]> {+ const unparsedFeedData = await parser.parseString(xmlData);++ return unparsedFeedData.items+ .map((item) => feedItemSchema.safeParse(item))+ .flatMap((item) => (item.success ? [item.data] : []));+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 801b2c99..a59ae0f2 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -971,6 +971,9 @@ importers:tsdown:specifier: ^0.12.9version: 0.12.9([email protected])+ vitest:+ specifier: ^3.2.4+ version: 3.2.4(@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])docs:dependencies:@@ -33797,6 +33800,50 @@ snapshots:tsx: 4.20.3yaml: 2.8.0+ [email protected](@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]):+ dependencies:+ '@types/chai': 5.2.2+ '@vitest/expect': 3.2.4+ '@vitest/mocker': 3.2.4([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))+ '@vitest/pretty-format': 3.2.4+ '@vitest/runner': 3.2.4+ '@vitest/snapshot': 3.2.4+ '@vitest/spy': 3.2.4+ '@vitest/utils': 3.2.4+ chai: 5.2.1+ debug: 4.4.1([email protected])+ expect-type: 1.2.2+ magic-string: 0.30.17+ pathe: 2.0.3+ picomatch: 4.0.2+ std-env: 3.9.0+ tinybench: 2.9.0+ tinyexec: 0.3.2+ tinyglobby: 0.2.14+ tinypool: 1.1.1+ tinyrainbow: 2.0.0+ vite: 7.0.6(@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])+ vite-node: 3.2.4(@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])+ why-is-node-running: 2.3.0+ optionalDependencies:+ '@types/debug': 4.1.12+ '@types/node': 24.10.3+ happy-dom: 20.0.8+ jsdom: 24.1.3+ transitivePeerDependencies:+ - jiti+ - less+ - lightningcss+ - msw+ - sass+ - sass-embedded+ - stylus+ - sugarss+ - supports-color+ - terser+ - tsx+ - yaml+[email protected](@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected]):dependencies:'@types/chai': 5.2.2