Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

fix(workers): handle TWZ RSS category objects (#2589)

T.J. Tarazevits645f2f0b2aaa
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 13500f20..199fe49b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -64,6 +64,10 @@ jobs:
working-directory: packages/trpc
run: pnpm test
+ - name: Workers Tests
+ working-directory: apps/workers
+ run: pnpm test
+
- name: E2E Tests
working-directory: packages/e2e_tests
run: pnpm test
diff --git a/apps/workers/package.json b/apps/workers/package.json
index 5c7277f9..7fbcd06f 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -62,7 +62,8 @@
"@types/archiver": "^7.0.0",
"@types/jsdom": "^21.1.6",
"@types/node-cron": "^3.0.11",
- "tsdown": "^0.12.9"
+ "tsdown": "^0.12.9",
+ "vitest": "^3.2.4"
},
"scripts": {
"start": "tsx watch index.ts",
@@ -73,6 +74,8 @@
"lint:fix": "oxlint . --fix",
"format": "oxfmt --check .",
"format:fix": "oxfmt .",
+ "test": "vitest run",
+ "test:watch": "vitest",
"typecheck": "tsc --noEmit"
}
}
diff --git a/apps/workers/workers/feedWorker.ts b/apps/workers/workers/feedWorker.ts
index eed7ccb1..0adba9f7 100644
--- a/apps/workers/workers/feedWorker.ts
+++ b/apps/workers/workers/feedWorker.ts
@@ -2,10 +2,8 @@ import { and, eq, inArray } from "drizzle-orm";
import { workerStatsCounter } from "metrics";
import { fetchWithProxy } from "network";
import cron from "node-cron";
-import Parser from "rss-parser";
import { buildImpersonatingTRPCClient } from "trpc";
import { withWorkerTracing } from "workerTracing";
-import { z } from "zod";
import type { ZFeedRequestSchema } from "@karakeep/shared-server";
import { db } from "@karakeep/db";
@@ -15,6 +13,8 @@ import logger from "@karakeep/shared/logger";
import { DequeuedJob, getQueueClient } from "@karakeep/shared/queueing";
import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
+import { parseFeedItems } from "./utils/feedParser";
+
/**
* Deterministically maps a feed ID to a minute offset within the hour (0-59).
* This ensures feeds are spread evenly across the hour based on their ID.
@@ -178,26 +178,7 @@ async function run(req: DequeuedJob<ZFeedRequestSchema>) {
`[feed][${jobId}] Successfully fetched feed "${feed.name}" (${feed.id}) ...`,
);
- const parser = new Parser({
- customFields: {
- item: ["id"],
- },
- });
- const unparseFeedData = await parser.parseString(xmlData);
-
- // Apparently, we can't trust the output of the xml parser. So let's do our own type
- // validation.
- const feedItemsSchema = z.object({
- id: z.coerce.string(),
- link: z.string().optional(),
- guid: z.string().optional(),
- title: z.string().optional(),
- categories: z.array(z.string()).optional(),
- });
-
- const feedItems = unparseFeedData.items
- .map((i) => feedItemsSchema.safeParse(i))
- .flatMap((i) => (i.success ? [i.data] : []));
+ const feedItems = await parseFeedItems(xmlData);
logger.info(
`[feed][${jobId}] Found ${feedItems.length} entries in feed "${feed.name}" (${feed.id}) ...`,
@@ -208,11 +189,6 @@ async function run(req: DequeuedJob<ZFeedRequestSchema>) {
return;
}
- // For feeds that don't have guids, use the link as the id
- feedItems.forEach((item) => {
- item.guid = item.guid ?? item.id ?? item.link;
- });
-
const exitingEntries = await db.query.rssFeedImportsTable.findMany({
where: and(
eq(rssFeedImportsTable.rssFeedId, feed.id),
diff --git a/apps/workers/workers/utils/__fixtures__/twz-feed.xml b/apps/workers/workers/utils/__fixtures__/twz-feed.xml
new file mode 100644
index 00000000..4e823709
--- /dev/null
+++ b/apps/workers/workers/utils/__fixtures__/twz-feed.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss
+ xmlns:content="http://purl.org/rss/1.0/modules/content/"
+ xmlns:dc="http://purl.org/dc/elements/1.1/"
+ version="2.0"
+>
+ <channel>
+ <title>The War Zone</title>
+ <link>https://www.twz.com</link>
+ <description><![CDATA[A strong offense for the world of defense.]]></description>
+ <item>
+ <title><![CDATA[Test TWZ article]]></title>
+ <description><![CDATA[<p>Test description.</p>]]></description>
+ <link>https://www.twz.com/sea/test-article</link>
+ <guid isPermaLink="false">https://www.twz.com/?p=12345</guid>
+ <pubDate>Tue, 14 Jan 2026 10:00:00 -0500</pubDate>
+ <category domain="https://www.twz.com/category/sea">Sea</category>
+ <category domain="https://www.twz.com/category/news-features">News &amp; Features</category>
+ <content:encoded><![CDATA[<p>Test content.</p>]]></content:encoded>
+ <dc:creator><![CDATA[Test Author]]></dc:creator>
+ </item>
+ </channel>
+</rss>
diff --git a/apps/workers/workers/utils/feedParser.test.ts b/apps/workers/workers/utils/feedParser.test.ts
new file mode 100644
index 00000000..e36761b7
--- /dev/null
+++ b/apps/workers/workers/utils/feedParser.test.ts
@@ -0,0 +1,50 @@
+import { readFile } from "node:fs/promises";
+
+import { describe, expect, test } from "vitest";
+
+import { parseFeedItems } from "./feedParser";
+
+describe("parseFeedItems", () => {
+ test("parses TWZ-style RSS items without dropping them", async () => {
+ const xmlData = await readFile(
+ new URL("./__fixtures__/twz-feed.xml", import.meta.url),
+ "utf8",
+ );
+
+ const items = await parseFeedItems(xmlData);
+
+ expect(items).toHaveLength(1);
+ expect(items[0]).toMatchObject({
+ guid: "https://www.twz.com/?p=12345",
+ link: "https://www.twz.com/sea/test-article",
+ title: "Test TWZ article",
+ categories: ["Sea", "News & Features"],
+ });
+ });
+
+ test("falls back to guid when feeds do not provide an item id", async () => {
+ const items = await parseFeedItems(`
+ <?xml version="1.0" encoding="UTF-8"?>
+ <rss version="2.0">
+ <channel>
+ <title>Test Feed</title>
+ <link>https://example.com</link>
+ <description>Test</description>
+ <item>
+ <guid isPermaLink="false">guid-1</guid>
+ <link>https://example.com/post-1</link>
+ <title>Post 1</title>
+ </item>
+ </channel>
+ </rss>
+ `);
+
+ expect(items).toHaveLength(1);
+ expect(items[0]).toMatchObject({
+ guid: "guid-1",
+ link: "https://example.com/post-1",
+ title: "Post 1",
+ });
+ expect(items[0].id).toBeUndefined();
+ });
+});
diff --git a/apps/workers/workers/utils/feedParser.ts b/apps/workers/workers/utils/feedParser.ts
new file mode 100644
index 00000000..4b485999
--- /dev/null
+++ b/apps/workers/workers/utils/feedParser.ts
@@ -0,0 +1,42 @@
+import Parser from "rss-parser";
+import { z } from "zod";
+
+const parser = new Parser({
+ customFields: {
+ item: ["id"],
+ },
+});
+
+const categorySchema = z
+ .union([z.string(), z.object({ _: z.string() })])
+ .transform((c) => (typeof c === "string" ? c : c._));
+
+const optionalStringSchema = z.preprocess(
+ (value) => (typeof value === "string" ? value : undefined),
+ z.string().optional(),
+);
+
+const feedItemSchema = z
+ .object({
+ id: optionalStringSchema,
+ link: z.string().optional(),
+ guid: z.string().optional(),
+ title: z.string().optional(),
+ categories: z.array(categorySchema).optional(),
+ })
+ .transform((item) => ({
+ ...item,
+ guid: item.guid ?? item.id ?? item.link,
+ }));
+
+export type ParsedFeedItem = z.infer<typeof feedItemSchema>;
+
+export async function parseFeedItems(
+ xmlData: string,
+): Promise<ParsedFeedItem[]> {
+ const unparsedFeedData = await parser.parseString(xmlData);
+
+ return unparsedFeedData.items
+ .map((item) => feedItemSchema.safeParse(item))
+ .flatMap((item) => (item.success ? [item.data] : []));
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 801b2c99..a59ae0f2 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -971,6 +971,9 @@ importers:
tsdown:
specifier: ^0.12.9
version: 0.12.9([email protected])
+ vitest:
+ specifier: ^3.2.4
docs:
dependencies:
@@ -33797,6 +33800,50 @@ snapshots:
tsx: 4.20.3
yaml: 2.8.0
+ dependencies:
+ '@types/chai': 5.2.2
+ '@vitest/expect': 3.2.4
+ '@vitest/pretty-format': 3.2.4
+ '@vitest/runner': 3.2.4
+ '@vitest/snapshot': 3.2.4
+ '@vitest/spy': 3.2.4
+ '@vitest/utils': 3.2.4
+ chai: 5.2.1
+ debug: 4.4.1([email protected])
+ expect-type: 1.2.2
+ magic-string: 0.30.17
+ pathe: 2.0.3
+ picomatch: 4.0.2
+ std-env: 3.9.0
+ tinybench: 2.9.0
+ tinyexec: 0.3.2
+ tinyglobby: 0.2.14
+ tinypool: 1.1.1
+ tinyrainbow: 2.0.0
+ why-is-node-running: 2.3.0
+ optionalDependencies:
+ '@types/debug': 4.1.12
+ '@types/node': 24.10.3
+ happy-dom: 20.0.8
+ jsdom: 24.1.3
+ transitivePeerDependencies:
+ - jiti
+ - less
+ - lightningcss
+ - msw
+ - sass
+ - sass-embedded
+ - stylus
+ - sugarss
+ - supports-color
+ - terser
+ - tsx
+ - yaml
+
dependencies:
'@types/chai': 5.2.2