Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

feat: export lists in backups and exports (#2484)

Mohamed Bassem0c458ba12de9
diff --git a/apps/web/app/api/bookmarks/export/route.tsx b/apps/web/app/api/bookmarks/export/route.tsx
index ad309877..6c40b00b 100644
--- a/apps/web/app/api/bookmarks/export/route.tsx
+++ b/apps/web/app/api/bookmarks/export/route.tsx
@@ -1,9 +1,19 @@
import { NextRequest } from "next/server";
-import { api, createContextFromRequest } from "@/server/api/client";
+import {
+ createContextFromRequest,
+ createTrcpClientFromCtx,
+} from "@/server/api/client";
+import { and, eq, inArray } from "drizzle-orm";
import { z } from "zod";
+import { db } from "@karakeep/db";
+import {
+ bookmarksInLists,
+ bookmarks as bookmarksTable,
+} from "@karakeep/db/schema";
import {
toExportFormat,
+ toExportListFormat,
toNetscapeFormat,
zExportSchema,
} from "@karakeep/shared/import-export";
@@ -24,11 +34,13 @@ export async function GET(request: NextRequest) {
includeContent: true,
};
- let resp = await api.bookmarks.getBookmarks(req);
+ const caller = createTrcpClientFromCtx(() => ctx);
+
+ let resp = await caller.bookmarks.getBookmarks(req);
let bookmarks = resp.bookmarks;
while (resp.nextCursor) {
- resp = await api.bookmarks.getBookmarks({
+ resp = await caller.bookmarks.getBookmarks({
...req,
cursor: resp.nextCursor,
});
@@ -36,11 +48,45 @@ export async function GET(request: NextRequest) {
}
if (format === "json") {
- // Default JSON format
+ // Fetch lists and bookmark-to-list memberships
+ const listsResp = await caller.lists.list();
+ const ownedLists = listsResp.lists.filter((l) => l.userRole === "owner");
+
+ const manualLists = ownedLists.filter((l) => l.type === "manual");
+ const manualListIds = manualLists.map((l) => l.id);
+
+ let memberships: { bookmarkId: string; listId: string }[] = [];
+ if (manualListIds.length > 0) {
+ memberships = await db
+ .select({
+ bookmarkId: bookmarksInLists.bookmarkId,
+ listId: bookmarksInLists.listId,
+ })
+ .from(bookmarksInLists)
+ .innerJoin(
+ bookmarksTable,
+ eq(bookmarksTable.id, bookmarksInLists.bookmarkId),
+ )
+ .where(
+ and(
+ inArray(bookmarksInLists.listId, manualListIds),
+ eq(bookmarksTable.userId, ctx.user.id),
+ ),
+ );
+ }
+
+ const bookmarkListMap = new Map<string, string[]>();
+ for (const m of memberships) {
+ const existing = bookmarkListMap.get(m.bookmarkId) ?? [];
+ existing.push(m.listId);
+ bookmarkListMap.set(m.bookmarkId, existing);
+ }
+
const exportData: z.infer<typeof zExportSchema> = {
bookmarks: bookmarks
- .map(toExportFormat)
+ .map((b) => toExportFormat(b, bookmarkListMap.get(b.id) ?? []))
.filter((b) => b.content !== null),
+ lists: ownedLists.map(toExportListFormat),
};
return new Response(JSON.stringify(exportData), {
diff --git a/apps/web/lib/hooks/useBookmarkImport.ts b/apps/web/lib/hooks/useBookmarkImport.ts
index 35c04c1b..a8eae4fd 100644
--- a/apps/web/lib/hooks/useBookmarkImport.ts
+++ b/apps/web/lib/hooks/useBookmarkImport.ts
@@ -52,8 +52,8 @@ export function useBookmarkImport() {
// First, parse the file to count bookmarks
const textContent = await file.text();
- const parsedBookmarks = parseImportFile(source, textContent);
- const bookmarkCount = parsedBookmarks.length;
+ const parsedImport = parseImportFile(source, textContent);
+ const bookmarkCount = parsedImport.bookmarks.length;
// Check quota before proceeding
if (bookmarkCount > 0) {
@@ -95,7 +95,7 @@ export function useBookmarkImport() {
{
// Use a custom parser to avoid re-parsing the file
parsers: {
- [source]: () => parsedBookmarks,
+ [source]: () => parsedImport,
},
},
);
diff --git a/apps/workers/workers/backupWorker.ts b/apps/workers/workers/backupWorker.ts
index 01f54b28..f4ea7bab 100644
--- a/apps/workers/workers/backupWorker.ts
+++ b/apps/workers/workers/backupWorker.ts
@@ -5,21 +5,30 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { createId } from "@paralleldrive/cuid2";
import archiver from "archiver";
-import { eq } from "drizzle-orm";
+import { and, eq, inArray } from "drizzle-orm";
import { workerStatsCounter } from "metrics";
import cron from "node-cron";
import { withWorkerTracing } from "workerTracing";
import type { ZBackupRequest } from "@karakeep/shared-server";
import { db } from "@karakeep/db";
-import { assets, AssetTypes, users } from "@karakeep/db/schema";
+import {
+ assets,
+ AssetTypes,
+ bookmarksInLists,
+ users,
+} from "@karakeep/db/schema";
import { BackupQueue, QuotaService } from "@karakeep/shared-server";
import { saveAssetFromFile } from "@karakeep/shared/assetdb";
-import { toExportFormat } from "@karakeep/shared/import-export";
+import {
+ toExportFormat,
+ toExportListFormat,
+} from "@karakeep/shared/import-export";
import logger from "@karakeep/shared/logger";
import { DequeuedJob, getQueueClient } from "@karakeep/shared/queueing";
import { AuthedContext } from "@karakeep/trpc";
import { Backup } from "@karakeep/trpc/models/backups";
+import { List } from "@karakeep/trpc/models/lists";
import { buildImpersonatingAuthedContext } from "../trpc";
import { fetchBookmarksInBatches } from "./utils/fetchBookmarks";
@@ -302,6 +311,16 @@ async function streamBookmarksToJsonFile(
outputPath: string,
jobId: string,
): Promise<number> {
+ // Pre-fetch list definitions (small data set)
+ const allLists = await List.getAllOwned(ctx);
+ const exportedLists = allLists.map((l) =>
+ toExportListFormat(l.asZBookmarkList()),
+ );
+
+ const manualListIds = allLists
+ .filter((l) => l.asZBookmarkList().type === "manual")
+ .map((l) => l.id);
+
return new Promise((resolve, reject) => {
const writeStream = createWriteStream(outputPath, { encoding: "utf-8" });
let bookmarkCount = 0;
@@ -309,14 +328,42 @@ async function streamBookmarksToJsonFile(
writeStream.on("error", reject);
- // Start JSON structure
- writeStream.write('{"bookmarks":[');
+ // Start JSON structure with lists first, then bookmarks
+ writeStream.write('{"lists":');
+ writeStream.write(JSON.stringify(exportedLists));
+ writeStream.write(',"bookmarks":[');
(async () => {
try {
for await (const batch of fetchBookmarksInBatches(ctx, 1000)) {
+ // Fetch memberships for this batch only to keep memory bounded
+ const batchBookmarkIds = batch.map((b) => b.id);
+ const bookmarkListMap = new Map<string, string[]>();
+ if (manualListIds.length > 0 && batchBookmarkIds.length > 0) {
+ const memberships = await ctx.db
+ .select({
+ bookmarkId: bookmarksInLists.bookmarkId,
+ listId: bookmarksInLists.listId,
+ })
+ .from(bookmarksInLists)
+ .where(
+ and(
+ inArray(bookmarksInLists.listId, manualListIds),
+ inArray(bookmarksInLists.bookmarkId, batchBookmarkIds),
+ ),
+ );
+ for (const m of memberships) {
+ const existing = bookmarkListMap.get(m.bookmarkId) ?? [];
+ existing.push(m.listId);
+ bookmarkListMap.set(m.bookmarkId, existing);
+ }
+ }
+
for (const bookmark of batch) {
- const exported = toExportFormat(bookmark);
+ const exported = toExportFormat(
+ bookmark,
+ bookmarkListMap.get(bookmark.id) ?? [],
+ );
if (exported.content !== null) {
// Add comma separator for all items except the first
if (!isFirst) {
diff --git a/packages/e2e_tests/tests/api/backups.test.ts b/packages/e2e_tests/tests/api/backups.test.ts
index 51c16c5e..3df823ad 100644
--- a/packages/e2e_tests/tests/api/backups.test.ts
+++ b/packages/e2e_tests/tests/api/backups.test.ts
@@ -169,6 +169,27 @@ describe("Backups API", () => {
bookmarks.push(bookmark!);
}
+ // Step 1b: Create a list and add first two bookmarks to it
+ const { data: listData } = await client.POST("/lists", {
+ body: {
+ name: "Backup Test List",
+ icon: "πŸ“‹",
+ },
+ });
+ expect(listData).toBeDefined();
+ const listId = listData!.id;
+
+ await client.PUT("/lists/{listId}/bookmarks/{bookmarkId}", {
+ params: {
+ path: { listId, bookmarkId: bookmarks[0].id },
+ },
+ });
+ await client.PUT("/lists/{listId}/bookmarks/{bookmarkId}", {
+ params: {
+ path: { listId, bookmarkId: bookmarks[1].id },
+ },
+ });
+
// Step 2: Trigger a backup
const { data: createdBackup, response: createResponse } =
await client.POST("/backups");
@@ -273,6 +294,31 @@ describe("Backups API", () => {
expect(firstBookmark).toHaveProperty("content");
expect(firstBookmark.content).toHaveProperty("type");
+ // Validate lists in backup
+ expect(backupData.lists).toBeDefined();
+ expect(Array.isArray(backupData.lists)).toBe(true);
+ const exportedList = backupData.lists.find(
+ (l: { name: string }) => l.name === "Backup Test List",
+ );
+ expect(exportedList).toBeDefined();
+ expect(exportedList.icon).toBe("πŸ“‹");
+ expect(exportedList.type).toBe("manual");
+ expect(exportedList.id).toBe(listId);
+
+ // Validate bookmark-to-list memberships
+ const bm1 = backupData.bookmarks.find(
+ (b: { title: string }) => b.title === "Test Bookmark 1",
+ );
+ const bm2 = backupData.bookmarks.find(
+ (b: { title: string }) => b.title === "Test Bookmark 2",
+ );
+ const bm3 = backupData.bookmarks.find(
+ (b: { title: string }) => b.title === "Test Bookmark 3",
+ );
+ expect(bm1.lists).toContain(listId);
+ expect(bm2.lists).toContain(listId);
+ expect(bm3.lists).toEqual([]);
+
// Step 7: Verify the backup appears in the list with updated status
const { data: backupsData } = await client.GET("/backups");
const listedBackup = backupsData!.backups.find((b) => b.id === backupId);
diff --git a/packages/shared/import-export/exporters.ts b/packages/shared/import-export/exporters.ts
index 967937a4..f7a2581a 100644
--- a/packages/shared/import-export/exporters.ts
+++ b/packages/shared/import-export/exporters.ts
@@ -1,11 +1,23 @@
import { z } from "zod";
import { BookmarkTypes, ZBookmark } from "../types/bookmarks";
+import { ZBookmarkList } from "../types/lists";
+
+export const zExportListSchema = z.object({
+ id: z.string(),
+ name: z.string(),
+ description: z.string().nullable(),
+ icon: z.string(),
+ type: z.enum(["manual", "smart"]),
+ query: z.string().nullable(),
+ parentId: z.string().nullable(),
+});
export const zExportBookmarkSchema = z.object({
createdAt: z.number(),
title: z.string().nullable(),
tags: z.array(z.string()),
+ lists: z.array(z.string()).optional().default([]),
content: z
.discriminatedUnion("type", [
z.object({
@@ -24,10 +36,12 @@ export const zExportBookmarkSchema = z.object({
export const zExportSchema = z.object({
bookmarks: z.array(zExportBookmarkSchema),
+ lists: z.array(zExportListSchema).optional().default([]),
});
export function toExportFormat(
bookmark: ZBookmark,
+ listIds?: string[],
): z.infer<typeof zExportBookmarkSchema> {
let content = null;
switch (bookmark.content.type) {
@@ -55,12 +69,27 @@ export function toExportFormat(
? (bookmark.content.title ?? null)
: null),
tags: bookmark.tags.map((t) => t.name),
+ lists: listIds ?? [],
content,
note: bookmark.note ?? null,
archived: bookmark.archived,
};
}
+export function toExportListFormat(
+ list: ZBookmarkList,
+): z.infer<typeof zExportListSchema> {
+ return {
+ id: list.id,
+ name: list.name,
+ description: list.description ?? null,
+ icon: list.icon,
+ type: list.type,
+ query: list.query ?? null,
+ parentId: list.parentId,
+ };
+}
+
export function toNetscapeFormat(bookmarks: ZBookmark[]): string {
const header = `<!DOCTYPE NETSCAPE-Bookmark-file-1>
<!-- This is an automatically generated file.
diff --git a/packages/shared/import-export/importer.test.ts b/packages/shared/import-export/importer.test.ts
index f097f8d5..ff15953b 100644
--- a/packages/shared/import-export/importer.test.ts
+++ b/packages/shared/import-export/importer.test.ts
@@ -10,45 +10,51 @@ const fakeFile = {
describe("importBookmarksFromFile", () => {
it("creates root list, folders and stages bookmarks with progress", async () => {
const parsers = {
- pocket: vi.fn().mockReturnValue([
- {
- title: "GitHub Repository",
- content: { type: "link", url: "https://github.com/example/repo" },
- tags: ["dev", "github"],
- addDate: 100,
- paths: [["Development", "Projects"]],
- },
- {
- title: "My Notes",
- content: { type: "text", text: "Important notes about the project" },
- tags: ["notes"],
- addDate: 200,
- paths: [["Personal"]],
- notes: "Additional context",
- archived: true,
- },
- {
- title: "Blog Post",
- content: { type: "link", url: "https://example.com/blog" },
- tags: ["reading", "tech"],
- addDate: 300,
- paths: [["Reading", "Tech"]],
- },
- {
- title: "No Category Item",
- content: { type: "link", url: "https://example.com/misc" },
- tags: [],
- addDate: 400,
- paths: [],
- },
- {
- title: "Duplicate URL Test",
- content: { type: "link", url: "https://github.com/example/repo" },
- tags: ["duplicate"],
- addDate: 50, // Earlier date
- paths: [["Development", "Duplicates"]],
- },
- ]),
+ pocket: vi.fn().mockReturnValue({
+ bookmarks: [
+ {
+ title: "GitHub Repository",
+ content: { type: "link", url: "https://github.com/example/repo" },
+ tags: ["dev", "github"],
+ addDate: 100,
+ paths: [["Development", "Projects"]],
+ },
+ {
+ title: "My Notes",
+ content: {
+ type: "text",
+ text: "Important notes about the project",
+ },
+ tags: ["notes"],
+ addDate: 200,
+ paths: [["Personal"]],
+ notes: "Additional context",
+ archived: true,
+ },
+ {
+ title: "Blog Post",
+ content: { type: "link", url: "https://example.com/blog" },
+ tags: ["reading", "tech"],
+ addDate: 300,
+ paths: [["Reading", "Tech"]],
+ },
+ {
+ title: "No Category Item",
+ content: { type: "link", url: "https://example.com/misc" },
+ tags: [],
+ addDate: 400,
+ paths: [],
+ },
+ {
+ title: "Duplicate URL Test",
+ content: { type: "link", url: "https://github.com/example/repo" },
+ tags: ["duplicate"],
+ addDate: 50, // Earlier date
+ paths: [["Development", "Duplicates"]],
+ },
+ ],
+ lists: [],
+ }),
};
const createdLists: { name: string; icon: string; parentId?: string }[] =
@@ -157,7 +163,9 @@ describe("importBookmarksFromFile", () => {
});
it("returns zero counts and null rootListId when no bookmarks", async () => {
- const parsers = { html: vi.fn().mockReturnValue([]) };
+ const parsers = {
+ html: vi.fn().mockReturnValue({ bookmarks: [], lists: [] }),
+ };
const res = await importBookmarksFromFile(
{
file: fakeFile,
@@ -181,29 +189,32 @@ describe("importBookmarksFromFile", () => {
it("stages all bookmarks successfully", async () => {
const parsers = {
- pocket: vi.fn().mockReturnValue([
- {
- title: "Bookmark 1",
- content: { type: "link", url: "https://example.com/1" },
- tags: ["tag1"],
- addDate: 100,
- paths: [["Category1"]],
- },
- {
- title: "Bookmark 2",
- content: { type: "link", url: "https://example.com/2" },
- tags: ["tag2"],
- addDate: 200,
- paths: [["Category2"]],
- },
- {
- title: "Bookmark 3",
- content: { type: "link", url: "https://example.com/3" },
- tags: ["tag3"],
- addDate: 300,
- paths: [["Category1"]],
- },
- ]),
+ pocket: vi.fn().mockReturnValue({
+ bookmarks: [
+ {
+ title: "Bookmark 1",
+ content: { type: "link", url: "https://example.com/1" },
+ tags: ["tag1"],
+ addDate: 100,
+ paths: [["Category1"]],
+ },
+ {
+ title: "Bookmark 2",
+ content: { type: "link", url: "https://example.com/2" },
+ tags: ["tag2"],
+ addDate: 200,
+ paths: [["Category2"]],
+ },
+ {
+ title: "Bookmark 3",
+ content: { type: "link", url: "https://example.com/3" },
+ tags: ["tag3"],
+ addDate: 300,
+ paths: [["Category1"]],
+ },
+ ],
+ lists: [],
+ }),
};
const createdLists: { name: string; icon: string; parentId?: string }[] =
@@ -281,29 +292,32 @@ describe("importBookmarksFromFile", () => {
it("stages bookmarks with different paths", async () => {
const parsers = {
- pocket: vi.fn().mockReturnValue([
- {
- title: "Bookmark 1",
- content: { type: "link", url: "https://example.com/1" },
- tags: ["tag1"],
- addDate: 100,
- paths: [["Path1"]],
- },
- {
- title: "Bookmark 2",
- content: { type: "link", url: "https://example.com/2" },
- tags: ["tag2"],
- addDate: 200,
- paths: [["Path2"]],
- },
- {
- title: "Bookmark 3",
- content: { type: "link", url: "https://example.com/3" },
- tags: ["tag3"],
- addDate: 300,
- paths: [["Path2"]],
- },
- ]),
+ pocket: vi.fn().mockReturnValue({
+ bookmarks: [
+ {
+ title: "Bookmark 1",
+ content: { type: "link", url: "https://example.com/1" },
+ tags: ["tag1"],
+ addDate: 100,
+ paths: [["Path1"]],
+ },
+ {
+ title: "Bookmark 2",
+ content: { type: "link", url: "https://example.com/2" },
+ tags: ["tag2"],
+ addDate: 200,
+ paths: [["Path2"]],
+ },
+ {
+ title: "Bookmark 3",
+ content: { type: "link", url: "https://example.com/3" },
+ tags: ["tag3"],
+ addDate: 300,
+ paths: [["Path2"]],
+ },
+ ],
+ lists: [],
+ }),
};
const createList = vi.fn(
@@ -365,6 +379,205 @@ describe("importBookmarksFromFile", () => {
expect(finalizeImportStaging).toHaveBeenCalledWith("session-1");
});
+ it("preserves separate list memberships when external list IDs differ", async () => {
+ const parsers = {
+ pocket: vi.fn().mockReturnValue({
+ bookmarks: [
+ {
+ title: "Bookmark 1",
+ content: { type: "link", url: "https://example.com/1" },
+ tags: [],
+ addDate: 100,
+ paths: [],
+ listExternalIds: ["child-1-id"],
+ },
+ {
+ title: "Bookmark 2",
+ content: { type: "link", url: "https://example.com/2" },
+ tags: [],
+ addDate: 200,
+ paths: [],
+ listExternalIds: ["child-2-id"],
+ },
+ ],
+ lists: [
+ {
+ externalId: "parent-id",
+ name: "Projects",
+ parentExternalId: null,
+ type: "manual",
+ },
+ {
+ externalId: "child-1-id",
+ name: "Inbox",
+ parentExternalId: "parent-id",
+ type: "manual",
+ },
+ {
+ externalId: "child-2-id",
+ name: "Inbox",
+ parentExternalId: "parent-id",
+ type: "manual",
+ },
+ ],
+ }),
+ };
+
+ let idCounter = 0;
+ const createdLists: { id: string; name: string; parentId?: string }[] = [];
+ const createList = vi.fn(
+ async (input: { name: string; icon: string; parentId?: string }) => {
+ const id = `list-${idCounter++}`;
+ createdLists.push({ id, name: input.name, parentId: input.parentId });
+ return { id };
+ },
+ );
+
+ const stagedBookmarks: StagedBookmark[] = [];
+ const stageImportedBookmarks = vi.fn(
+ async (input: {
+ importSessionId: string;
+ bookmarks: StagedBookmark[];
+ }) => {
+ stagedBookmarks.push(...input.bookmarks);
+ },
+ );
+
+ await importBookmarksFromFile(
+ {
+ file: fakeFile,
+ source: "pocket",
+ rootListName: "Imported",
+ deps: {
+ createList,
+ stageImportedBookmarks,
+ finalizeImportStaging: vi.fn(),
+ createImportSession: vi.fn(async () => ({ id: "session-1" })),
+ },
+ },
+ { parsers },
+ );
+
+ const projectsFolder = createdLists.find(
+ (list) => list.name === "Projects",
+ );
+ expect(projectsFolder).toBeDefined();
+
+ const duplicateFolders = createdLists.filter(
+ (list) => list.name === "Inbox" && list.parentId === projectsFolder?.id,
+ );
+ expect(duplicateFolders).toHaveLength(2);
+
+ const firstBookmark = stagedBookmarks.find(
+ (bookmark) => bookmark.url === "https://example.com/1",
+ );
+ const secondBookmark = stagedBookmarks.find(
+ (bookmark) => bookmark.url === "https://example.com/2",
+ );
+
+ expect(firstBookmark).toBeDefined();
+ expect(secondBookmark).toBeDefined();
+ expect(firstBookmark?.listIds[0]).not.toEqual(secondBookmark?.listIds[0]);
+ expect(duplicateFolders.map((list) => list.id)).toContain(
+ firstBookmark?.listIds[0],
+ );
+ expect(duplicateFolders.map((list) => list.id)).toContain(
+ secondBookmark?.listIds[0],
+ );
+ });
+
+ it("creates smart lists with their queries during import", async () => {
+ const parsers = {
+ karakeep: vi.fn().mockReturnValue({
+ bookmarks: [
+ {
+ title: "Bookmark 1",
+ content: { type: "link", url: "https://example.com/1" },
+ tags: [],
+ addDate: 100,
+ paths: [],
+ listExternalIds: ["manual-list-id"],
+ },
+ ],
+ lists: [
+ {
+ externalId: "manual-list-id",
+ name: "Manual",
+ icon: "⭐",
+ description: "Manual list description",
+ parentExternalId: null,
+ type: "manual",
+ },
+ {
+ externalId: "smart-list-id",
+ name: "Smart",
+ icon: "⚑",
+ description: "Smart list description",
+ parentExternalId: null,
+ type: "smart",
+ query: "tag:read-later",
+ },
+ ],
+ }),
+ };
+
+ const createdLists: {
+ name: string;
+ icon: string;
+ description?: string;
+ parentId?: string;
+ type?: "manual" | "smart";
+ query?: string;
+ }[] = [];
+
+ const createList = vi.fn(
+ async (input: {
+ name: string;
+ icon: string;
+ description?: string;
+ parentId?: string;
+ type?: "manual" | "smart";
+ query?: string;
+ }) => {
+ createdLists.push(input);
+ return {
+ id: `${input.parentId ? input.parentId + "/" : ""}${input.name}`,
+ };
+ },
+ );
+
+ await importBookmarksFromFile(
+ {
+ file: fakeFile,
+ source: "karakeep",
+ rootListName: "Imported",
+ deps: {
+ createList,
+ stageImportedBookmarks: vi.fn(async () => undefined),
+ finalizeImportStaging: vi.fn(),
+ createImportSession: vi.fn(async () => ({ id: "session-1" })),
+ },
+ },
+ { parsers },
+ );
+
+ expect(createdLists).toContainEqual({
+ name: "Smart",
+ parentId: "Imported",
+ icon: "⚑",
+ description: "Smart list description",
+ type: "smart",
+ query: "tag:read-later",
+ });
+
+ expect(createdLists).toContainEqual({
+ name: "Manual",
+ parentId: "Imported",
+ icon: "⭐",
+ description: "Manual list description",
+ });
+ });
+
it("handles HTML bookmarks with empty folder names", async () => {
const htmlContent = `<!DOCTYPE NETSCAPE-Bookmark-file-1>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
diff --git a/packages/shared/import-export/importer.ts b/packages/shared/import-export/importer.ts
index be24ca73..a629d39c 100644
--- a/packages/shared/import-export/importer.ts
+++ b/packages/shared/import-export/importer.ts
@@ -1,5 +1,5 @@
import { MAX_LIST_NAME_LENGTH } from "../types/lists";
-import { ImportSource, ParsedBookmark, parseImportFile } from "./parsers";
+import { ImportSource, ParsedImportFile, parseImportFile } from "./parsers";
export interface ImportCounts {
successes: number;
@@ -23,7 +23,10 @@ export interface ImportDeps {
createList: (input: {
name: string;
icon: string;
+ description?: string;
parentId?: string;
+ type?: "manual" | "smart";
+ query?: string;
}) => Promise<{ id: string }>;
stageImportedBookmarks: (input: {
importSessionId: string;
@@ -39,7 +42,7 @@ export interface ImportDeps {
export interface ImportOptions {
concurrencyLimit?: number;
parsers?: Partial<
- Record<ImportSource, (textContent: string) => ParsedBookmark[]>
+ Record<ImportSource, (textContent: string) => ParsedImportFile>
>;
}
@@ -68,9 +71,11 @@ export async function importBookmarksFromFile(
const { parsers } = options;
const textContent = await file.text();
- const parsedBookmarks = parsers?.[source]
+ const parsedImport = parsers?.[source]
? parsers[source]!(textContent)
: parseImportFile(source, textContent);
+ const parsedBookmarks = parsedImport.bookmarks;
+ const parsedLists = parsedImport.lists;
if (parsedBookmarks.length === 0) {
return {
counts: { successes: 0, failures: 0, alreadyExisted: 0, total: 0 },
@@ -87,34 +92,100 @@ export async function importBookmarksFromFile(
onProgress?.(0, parsedBookmarks.length);
+ const externalListIdToCreatedListId: Record<string, string> = {};
+ if (parsedLists.length > 0) {
+ const unresolvedLists = new Map(
+ parsedLists.map((list) => [list.externalId, list]),
+ );
+
+ while (unresolvedLists.size > 0) {
+ let createdAny = false;
+
+ for (const [externalId, list] of unresolvedLists) {
+ if (
+ list.parentExternalId &&
+ !externalListIdToCreatedListId[list.parentExternalId]
+ ) {
+ continue;
+ }
+
+ const parentId = list.parentExternalId
+ ? externalListIdToCreatedListId[list.parentExternalId]
+ : rootList.id;
+
+ const createdList = await deps.createList({
+ name: list.name.substring(0, MAX_LIST_NAME_LENGTH),
+ parentId,
+ icon: list.icon ?? "πŸ“",
+ description: list.description,
+ ...(list.type === "smart" && list.query
+ ? { type: "smart", query: list.query }
+ : {}),
+ });
+ externalListIdToCreatedListId[externalId] = createdList.id;
+ unresolvedLists.delete(externalId);
+ createdAny = true;
+ }
+
+ // Break cycles or unresolved parent references by attaching remaining
+ // lists to the import root.
+ if (!createdAny) {
+ for (const [externalId, list] of unresolvedLists) {
+ const createdList = await deps.createList({
+ name: list.name.substring(0, MAX_LIST_NAME_LENGTH),
+ parentId: rootList.id,
+ icon: list.icon ?? "πŸ“",
+ description: list.description,
+ ...(list.type === "smart" && list.query
+ ? { type: "smart", query: list.query }
+ : {}),
+ });
+ externalListIdToCreatedListId[externalId] = createdList.id;
+ }
+ unresolvedLists.clear();
+ }
+ }
+ }
+
const PATH_DELIMITER = "$$__$$";
+ const getPathKey = (parts: string[]) => parts.join(PATH_DELIMITER);
+ const bookmarksWithPathMembership = parsedBookmarks.filter(
+ (bookmark) =>
+ !bookmark.listExternalIds || bookmark.listExternalIds.length === 0,
+ );
// Build required paths
- const allRequiredPaths = new Set<string>();
- for (const bookmark of parsedBookmarks) {
+ const allRequiredPaths = new Map<string, string>();
+ for (const bookmark of bookmarksWithPathMembership) {
for (const path of bookmark.paths) {
if (path && path.length > 0) {
for (let i = 1; i <= path.length; i++) {
const subPath = path.slice(0, i);
- const pathKey = subPath.join(PATH_DELIMITER);
- allRequiredPaths.add(pathKey);
+ const pathKey = getPathKey(subPath);
+ const folderName = subPath[subPath.length - 1];
+ if (!allRequiredPaths.has(pathKey)) {
+ allRequiredPaths.set(pathKey, folderName);
+ }
}
}
}
}
- const allRequiredPathsArray = Array.from(allRequiredPaths).sort(
- (a, b) => a.split(PATH_DELIMITER).length - b.split(PATH_DELIMITER).length,
- );
+ const allRequiredPathsArray = Array.from(allRequiredPaths.entries())
+ .map(([pathKey, folderName]) => ({ pathKey, folderName }))
+ .sort(
+ (a, b) =>
+ a.pathKey.split(PATH_DELIMITER).length -
+ b.pathKey.split(PATH_DELIMITER).length,
+ );
const pathMap: Record<string, string> = { "": rootList.id };
- for (const pathKey of allRequiredPathsArray) {
+ for (const { pathKey, folderName } of allRequiredPathsArray) {
const parts = pathKey.split(PATH_DELIMITER);
const parentKey = parts.slice(0, -1).join(PATH_DELIMITER);
const parentId = pathMap[parentKey] || rootList.id;
- const folderName = parts[parts.length - 1];
const folderList = await deps.createList({
name: folderName.substring(0, MAX_LIST_NAME_LENGTH),
parentId,
@@ -127,7 +198,7 @@ export async function importBookmarksFromFile(
const bookmarksToStage: StagedBookmark[] = parsedBookmarks.map((bookmark) => {
// Convert paths to list IDs using pathMap
// If no paths, assign to root list
- const listIds =
+ const listIdsFromPaths =
bookmark.paths.length === 0
? [rootList.id]
: bookmark.paths
@@ -135,11 +206,26 @@ export async function importBookmarksFromFile(
if (path.length === 0) {
return rootList.id;
}
- const pathKey = path.join(PATH_DELIMITER);
+ const pathKey = getPathKey(path);
return pathMap[pathKey] || rootList.id;
})
.filter((id, index, arr) => arr.indexOf(id) === index); // dedupe
+ const externalListIds = bookmark.listExternalIds ?? [];
+ const listIdsFromExternalListIds =
+ externalListIds.length > 0
+ ? [
+ ...new Set(
+ externalListIds.map((id) => externalListIdToCreatedListId[id]),
+ ),
+ ].filter((id): id is string => Boolean(id))
+ : [];
+
+ const listIds =
+ listIdsFromExternalListIds.length > 0
+ ? listIdsFromExternalListIds
+ : listIdsFromPaths;
+
// Determine type and extract content appropriately
let type: "link" | "text" | "asset" = "link";
let url: string | undefined;
diff --git a/packages/shared/import-export/index.ts b/packages/shared/import-export/index.ts
index dd266e06..a6940306 100644
--- a/packages/shared/import-export/index.ts
+++ b/packages/shared/import-export/index.ts
@@ -1,4 +1,9 @@
export * from "./exporters";
export * from "./importer";
-export type { ImportSource, ParsedBookmark } from "./parsers";
+export type {
+ ImportSource,
+ ParsedBookmark,
+ ParsedImportFile,
+ ParsedImportList,
+} from "./parsers";
export { parseImportFile } from "./parsers";
diff --git a/packages/shared/import-export/parsers.test.ts b/packages/shared/import-export/parsers.test.ts
index 18502305..a35d7bb8 100644
--- a/packages/shared/import-export/parsers.test.ts
+++ b/packages/shared/import-export/parsers.test.ts
@@ -12,7 +12,9 @@ describe("parseNetscapeBookmarkFile", () => {
<DT><A HREF="https://example.com" ADD_DATE="1234567890">Example Site</A>
</DL><p>`;
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(1);
expect(result[0]).toMatchObject({
@@ -36,7 +38,9 @@ describe("parseNetscapeBookmarkFile", () => {
<DT><A HREF="https://example.com" ADD_DATE="1234567890" TAGS="tag1,tag2,tag3">Example Site</A>
</DL><p>`;
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(1);
expect(result[0].tags).toEqual(["tag1", "tag2", "tag3"]);
@@ -57,7 +61,9 @@ describe("parseNetscapeBookmarkFile", () => {
</DL><p>
</DL><p>`;
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(1);
expect(result[0]).toMatchObject({
@@ -85,7 +91,9 @@ describe("parseNetscapeBookmarkFile", () => {
</DL><p>
</DL><p>`;
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(1);
expect(result[0].paths).toEqual([["Named Folder", "Unnamed"]]);
@@ -108,7 +116,9 @@ describe("parseNetscapeBookmarkFile", () => {
</DL><p>
</DL><p>`;
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(3);
@@ -141,7 +151,9 @@ describe("parseNetscapeBookmarkFile", () => {
<DT><A HREF="https://example2.com" ADD_DATE="1234567891">Bookmark 2</A>
</DL><p>`;
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(2);
expect(result[0].paths).toEqual([[]]);
@@ -169,7 +181,9 @@ describe("parseNetscapeBookmarkFile", () => {
</DL><p>
</DL><p>`;
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(1);
expect(result[0].paths).toEqual([["Level1", "Level2", "Level3", "Level4"]]);
@@ -191,7 +205,9 @@ describe("parseNetscapeBookmarkFile", () => {
</DL><p>
</DL><p>`;
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(1);
expect(result[0]).toMatchObject({
@@ -218,7 +234,9 @@ describe("parseNetscapeBookmarkFile", () => {
// Note: The current parser doesn't extract DD notes, but this test
// documents the expected behavior if/when DD parsing is added
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(1);
expect(result[0].content).toMatchObject({
@@ -236,7 +254,9 @@ describe("parseNetscapeBookmarkFile", () => {
<DT><A HREF="https://example.com">No Date Bookmark</A>
</DL><p>`;
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(1);
expect(result[0].addDate).toBeUndefined();
@@ -251,7 +271,9 @@ describe("parseNetscapeBookmarkFile", () => {
<DT><A ADD_DATE="1234567890">No URL Bookmark</A>
</DL><p>`;
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(1);
expect(result[0].content).toBeUndefined();
@@ -271,7 +293,9 @@ describe("parseNetscapeBookmarkFile", () => {
<DT><A HREF="https://root2.com" ADD_DATE="1234567892">Root Bookmark 2</A>
</DL><p>`;
- const result = parseImportFile("html", html);
+ const parsed = parseImportFile("html", html);
+
+ const result = parsed.bookmarks;
expect(result).toHaveLength(3);
expect(result[0]).toMatchObject({
@@ -299,3 +323,141 @@ describe("parseNetscapeBookmarkFile", () => {
);
});
});
+
+describe("parseKarakeepBookmarkFile", () => {
+ it("keeps distinct identities for duplicate sibling list names", () => {
+ const json = JSON.stringify({
+ lists: [
+ {
+ id: "parent",
+ name: "Projects",
+ description: null,
+ icon: "πŸ“",
+ type: "manual",
+ query: null,
+ parentId: null,
+ },
+ {
+ id: "child-1",
+ name: "Inbox",
+ description: null,
+ icon: "πŸ“",
+ type: "manual",
+ query: null,
+ parentId: "parent",
+ },
+ {
+ id: "child-2",
+ name: "Inbox",
+ description: null,
+ icon: "πŸ“",
+ type: "manual",
+ query: null,
+ parentId: "parent",
+ },
+ ],
+ bookmarks: [
+ {
+ createdAt: 123,
+ title: "One",
+ tags: [],
+ lists: ["child-1"],
+ content: { type: "link", url: "https://one.example" },
+ note: null,
+ archived: false,
+ },
+ {
+ createdAt: 456,
+ title: "Two",
+ tags: [],
+ lists: ["child-2"],
+ content: { type: "link", url: "https://two.example" },
+ note: null,
+ archived: false,
+ },
+ ],
+ });
+
+ const result = parseImportFile("karakeep", json);
+
+ expect(result.lists).toEqual([
+ {
+ externalId: "parent",
+ name: "Projects",
+ icon: "πŸ“",
+ description: undefined,
+ parentExternalId: null,
+ type: "manual",
+ },
+ {
+ externalId: "child-1",
+ name: "Inbox",
+ icon: "πŸ“",
+ description: undefined,
+ parentExternalId: "parent",
+ type: "manual",
+ },
+ {
+ externalId: "child-2",
+ name: "Inbox",
+ icon: "πŸ“",
+ description: undefined,
+ parentExternalId: "parent",
+ type: "manual",
+ },
+ ]);
+
+ expect(result.bookmarks).toHaveLength(2);
+ expect(result.bookmarks[0].listExternalIds).toEqual(["child-1"]);
+ expect(result.bookmarks[1].listExternalIds).toEqual(["child-2"]);
+ });
+
+ it("preserves smart list query metadata", () => {
+ const json = JSON.stringify({
+ lists: [
+ {
+ id: "manual",
+ name: "Manual",
+ description: null,
+ icon: "πŸ“",
+ type: "manual",
+ query: null,
+ parentId: null,
+ },
+ {
+ id: "smart",
+ name: "Smart",
+ description: "Smart list description",
+ icon: "⚑",
+ type: "smart",
+ query: "tag:read-later",
+ parentId: null,
+ },
+ ],
+ bookmarks: [
+ {
+ createdAt: 123,
+ title: "One",
+ tags: [],
+ lists: ["manual", "smart"],
+ content: { type: "link", url: "https://one.example" },
+ note: null,
+ archived: false,
+ },
+ ],
+ });
+
+ const result = parseImportFile("karakeep", json);
+
+ expect(result.lists).toContainEqual({
+ externalId: "smart",
+ name: "Smart",
+ icon: "⚑",
+ description: "Smart list description",
+ parentExternalId: null,
+ type: "smart",
+ query: "tag:read-later",
+ });
+ expect(result.bookmarks[0].listExternalIds).toEqual(["manual"]);
+ });
+});
diff --git a/packages/shared/import-export/parsers.ts b/packages/shared/import-export/parsers.ts
index 24d85c80..b46d0e65 100644
--- a/packages/shared/import-export/parsers.ts
+++ b/packages/shared/import-export/parsers.ts
@@ -29,6 +29,23 @@ export interface ParsedBookmark {
notes?: string;
archived?: boolean;
paths: string[][];
+ // Optional list IDs from the source file (used with top-level `lists`).
+ listExternalIds?: string[];
+}
+
+export interface ParsedImportList {
+ externalId: string;
+ name: string;
+ icon?: string;
+ description?: string;
+ parentExternalId: string | null;
+ type: "manual" | "smart";
+ query?: string;
+}
+
+export interface ParsedImportFile {
+ bookmarks: ParsedBookmark[];
+ lists: ParsedImportList[];
}
function parseNetscapeBookmarkFile(textContent: string): ParsedBookmark[] {
@@ -161,7 +178,7 @@ function parseMatterBookmarkFile(textContent: string): ParsedBookmark[] {
});
}
-function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] {
+function parseKarakeepBookmarkFile(textContent: string): ParsedImportFile {
const parsed = zExportSchema.safeParse(JSON.parse(textContent));
if (!parsed.success) {
throw new Error(
@@ -169,7 +186,22 @@ function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] {
);
}
- return parsed.data.bookmarks.map((bookmark) => {
+ const exportedLists = parsed.data.lists ?? [];
+ const parsedLists: ParsedImportList[] = exportedLists.map((list) => ({
+ externalId: list.id,
+ name: list.name,
+ icon: list.icon,
+ description: list.description ?? undefined,
+ parentExternalId: list.parentId,
+ type: list.type,
+ query: list.type === "smart" ? (list.query ?? undefined) : undefined,
+ }));
+
+ const manualListIds = new Set(
+ exportedLists.filter((l) => l.type === "manual").map((l) => l.id),
+ );
+
+ const parsedBookmarks = parsed.data.bookmarks.map((bookmark) => {
let content = undefined;
if (bookmark.content?.type == BookmarkTypes.LINK) {
content = {
@@ -182,6 +214,7 @@ function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] {
text: bookmark.content.text,
};
}
+
return {
title: bookmark.title ?? "",
content,
@@ -189,9 +222,17 @@ function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] {
addDate: bookmark.createdAt,
notes: bookmark.note ?? undefined,
archived: bookmark.archived,
- paths: [], // TODO
+ paths: [],
+ listExternalIds: (bookmark.lists ?? []).filter((listId) =>
+ manualListIds.has(listId),
+ ),
};
});
+
+ return {
+ bookmarks: parsedBookmarks,
+ lists: parsedLists,
+ };
}
function parseOmnivoreBookmarkFile(textContent: string): ParsedBookmark[] {
@@ -429,6 +470,14 @@ function deduplicateBookmarks(bookmarks: ParsedBookmark[]): ParsedBookmark[] {
existing.tags = [...new Set([...existing.tags, ...bookmark.tags])];
// Merge paths
existing.paths = [...existing.paths, ...bookmark.paths];
+ if (existing.listExternalIds || bookmark.listExternalIds) {
+ existing.listExternalIds = [
+ ...new Set([
+ ...(existing.listExternalIds ?? []),
+ ...(bookmark.listExternalIds ?? []),
+ ]),
+ ];
+ }
const existingDate = existing.addDate ?? Infinity;
const newDate = bookmark.addDate ?? Infinity;
if (newDate < existingDate) {
@@ -460,7 +509,15 @@ function deduplicateBookmarks(bookmarks: ParsedBookmark[]): ParsedBookmark[] {
export function parseImportFile(
source: ImportSource,
textContent: string,
-): ParsedBookmark[] {
+): ParsedImportFile {
+ if (source === "karakeep") {
+ const parsed = parseKarakeepBookmarkFile(textContent);
+ return {
+ bookmarks: deduplicateBookmarks(parsed.bookmarks),
+ lists: parsed.lists,
+ };
+ }
+
let result: ParsedBookmark[];
switch (source) {
case "html":
@@ -472,9 +529,6 @@ export function parseImportFile(
case "matter":
result = parseMatterBookmarkFile(textContent);
break;
- case "karakeep":
- result = parseKarakeepBookmarkFile(textContent);
- break;
case "omnivore":
result = parseOmnivoreBookmarkFile(textContent);
break;
@@ -491,5 +545,5 @@ export function parseImportFile(
result = parseInstapaperBookmarkFile(textContent);
break;
}
- return deduplicateBookmarks(result);
+ return { bookmarks: deduplicateBookmarks(result), lists: [] };
}