A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
feat: export lists in backups and exports (#2484)
Mohamed Bassem0c458ba12de9
diff --git a/apps/web/app/api/bookmarks/export/route.tsx b/apps/web/app/api/bookmarks/export/route.tsxindex ad309877..6c40b00b 100644--- a/apps/web/app/api/bookmarks/export/route.tsx+++ b/apps/web/app/api/bookmarks/export/route.tsx@@ -1,9 +1,19 @@import { NextRequest } from "next/server";-import { api, createContextFromRequest } from "@/server/api/client";+import {+ createContextFromRequest,+ createTrcpClientFromCtx,+} from "@/server/api/client";+import { and, eq, inArray } from "drizzle-orm";import { z } from "zod";+import { db } from "@karakeep/db";+import {+ bookmarksInLists,+ bookmarks as bookmarksTable,+} from "@karakeep/db/schema";import {toExportFormat,+ toExportListFormat,toNetscapeFormat,zExportSchema,} from "@karakeep/shared/import-export";@@ -24,11 +34,13 @@ export async function GET(request: NextRequest) {includeContent: true,};- let resp = await api.bookmarks.getBookmarks(req);+ const caller = createTrcpClientFromCtx(() => ctx);++ let resp = await caller.bookmarks.getBookmarks(req);let bookmarks = resp.bookmarks;while (resp.nextCursor) {- resp = await api.bookmarks.getBookmarks({+ resp = await caller.bookmarks.getBookmarks({...req,cursor: resp.nextCursor,});@@ -36,11 +48,45 @@ export async function GET(request: NextRequest) {}if (format === "json") {- // Default JSON format+ // Fetch lists and bookmark-to-list memberships+ const listsResp = await caller.lists.list();+ const ownedLists = listsResp.lists.filter((l) => l.userRole === "owner");++ const manualLists = ownedLists.filter((l) => l.type === "manual");+ const manualListIds = manualLists.map((l) => l.id);++ let memberships: { bookmarkId: string; listId: string }[] = [];+ if (manualListIds.length > 0) {+ memberships = await db+ .select({+ bookmarkId: bookmarksInLists.bookmarkId,+ listId: bookmarksInLists.listId,+ })+ .from(bookmarksInLists)+ .innerJoin(+ bookmarksTable,+ eq(bookmarksTable.id, bookmarksInLists.bookmarkId),+ )+ .where(+ and(+ inArray(bookmarksInLists.listId, manualListIds),+ eq(bookmarksTable.userId, ctx.user.id),+ ),+ );+ }++ const bookmarkListMap = new Map<string, string[]>();+ for (const m of memberships) {+ const existing = bookmarkListMap.get(m.bookmarkId) ?? [];+ existing.push(m.listId);+ bookmarkListMap.set(m.bookmarkId, existing);+ }+const exportData: z.infer<typeof zExportSchema> = {bookmarks: bookmarks- .map(toExportFormat)+ .map((b) => toExportFormat(b, bookmarkListMap.get(b.id) ?? [])).filter((b) => b.content !== null),+ lists: ownedLists.map(toExportListFormat),};return new Response(JSON.stringify(exportData), {diff --git a/apps/web/lib/hooks/useBookmarkImport.ts b/apps/web/lib/hooks/useBookmarkImport.tsindex 35c04c1b..a8eae4fd 100644--- a/apps/web/lib/hooks/useBookmarkImport.ts+++ b/apps/web/lib/hooks/useBookmarkImport.ts@@ -52,8 +52,8 @@ export function useBookmarkImport() {// First, parse the file to count bookmarksconst textContent = await file.text();- const parsedBookmarks = parseImportFile(source, textContent);- const bookmarkCount = parsedBookmarks.length;+ const parsedImport = parseImportFile(source, textContent);+ const bookmarkCount = parsedImport.bookmarks.length;// Check quota before proceedingif (bookmarkCount > 0) {@@ -95,7 +95,7 @@ export function useBookmarkImport() {{// Use a custom parser to avoid re-parsing the fileparsers: {- [source]: () => parsedBookmarks,+ [source]: () => parsedImport,},},);diff --git a/apps/workers/workers/backupWorker.ts b/apps/workers/workers/backupWorker.tsindex 01f54b28..f4ea7bab 100644--- a/apps/workers/workers/backupWorker.ts+++ b/apps/workers/workers/backupWorker.ts@@ -5,21 +5,30 @@ import { tmpdir } from "node:os";import { join } from "node:path";import { createId } from "@paralleldrive/cuid2";import archiver from "archiver";-import { eq } from "drizzle-orm";+import { and, eq, inArray } from "drizzle-orm";import { workerStatsCounter } from "metrics";import cron from "node-cron";import { withWorkerTracing } from "workerTracing";import type { ZBackupRequest } from "@karakeep/shared-server";import { db } from "@karakeep/db";-import { assets, AssetTypes, users } from "@karakeep/db/schema";+import {+ assets,+ AssetTypes,+ bookmarksInLists,+ users,+} from "@karakeep/db/schema";import { BackupQueue, QuotaService } from "@karakeep/shared-server";import { saveAssetFromFile } from "@karakeep/shared/assetdb";-import { toExportFormat } from "@karakeep/shared/import-export";+import {+ toExportFormat,+ toExportListFormat,+} from "@karakeep/shared/import-export";import logger from "@karakeep/shared/logger";import { DequeuedJob, getQueueClient } from "@karakeep/shared/queueing";import { AuthedContext } from "@karakeep/trpc";import { Backup } from "@karakeep/trpc/models/backups";+import { List } from "@karakeep/trpc/models/lists";import { buildImpersonatingAuthedContext } from "../trpc";import { fetchBookmarksInBatches } from "./utils/fetchBookmarks";@@ -302,6 +311,16 @@ async function streamBookmarksToJsonFile(outputPath: string,jobId: string,): Promise<number> {+ // Pre-fetch list definitions (small data set)+ const allLists = await List.getAllOwned(ctx);+ const exportedLists = allLists.map((l) =>+ toExportListFormat(l.asZBookmarkList()),+ );++ const manualListIds = allLists+ .filter((l) => l.asZBookmarkList().type === "manual")+ .map((l) => l.id);+return new Promise((resolve, reject) => {const writeStream = createWriteStream(outputPath, { encoding: "utf-8" });let bookmarkCount = 0;@@ -309,14 +328,42 @@ async function streamBookmarksToJsonFile(writeStream.on("error", reject);- // Start JSON structure- writeStream.write('{"bookmarks":[');+ // Start JSON structure with lists first, then bookmarks+ writeStream.write('{"lists":');+ writeStream.write(JSON.stringify(exportedLists));+ writeStream.write(',"bookmarks":[');(async () => {try {for await (const batch of fetchBookmarksInBatches(ctx, 1000)) {+ // Fetch memberships for this batch only to keep memory bounded+ const batchBookmarkIds = batch.map((b) => b.id);+ const bookmarkListMap = new Map<string, string[]>();+ if (manualListIds.length > 0 && batchBookmarkIds.length > 0) {+ const memberships = await ctx.db+ .select({+ bookmarkId: bookmarksInLists.bookmarkId,+ listId: bookmarksInLists.listId,+ })+ .from(bookmarksInLists)+ .where(+ and(+ inArray(bookmarksInLists.listId, manualListIds),+ inArray(bookmarksInLists.bookmarkId, batchBookmarkIds),+ ),+ );+ for (const m of memberships) {+ const existing = bookmarkListMap.get(m.bookmarkId) ?? [];+ existing.push(m.listId);+ bookmarkListMap.set(m.bookmarkId, existing);+ }+ }+for (const bookmark of batch) {- const exported = toExportFormat(bookmark);+ const exported = toExportFormat(+ bookmark,+ bookmarkListMap.get(bookmark.id) ?? [],+ );if (exported.content !== null) {// Add comma separator for all items except the firstif (!isFirst) {diff --git a/packages/e2e_tests/tests/api/backups.test.ts b/packages/e2e_tests/tests/api/backups.test.tsindex 51c16c5e..3df823ad 100644--- a/packages/e2e_tests/tests/api/backups.test.ts+++ b/packages/e2e_tests/tests/api/backups.test.ts@@ -169,6 +169,27 @@ describe("Backups API", () => {bookmarks.push(bookmark!);}+ // Step 1b: Create a list and add first two bookmarks to it+ const { data: listData } = await client.POST("/lists", {+ body: {+ name: "Backup Test List",+ icon: "π",+ },+ });+ expect(listData).toBeDefined();+ const listId = listData!.id;++ await client.PUT("/lists/{listId}/bookmarks/{bookmarkId}", {+ params: {+ path: { listId, bookmarkId: bookmarks[0].id },+ },+ });+ await client.PUT("/lists/{listId}/bookmarks/{bookmarkId}", {+ params: {+ path: { listId, bookmarkId: bookmarks[1].id },+ },+ });+// Step 2: Trigger a backupconst { data: createdBackup, response: createResponse } =await client.POST("/backups");@@ -273,6 +294,31 @@ describe("Backups API", () => {expect(firstBookmark).toHaveProperty("content");expect(firstBookmark.content).toHaveProperty("type");+ // Validate lists in backup+ expect(backupData.lists).toBeDefined();+ expect(Array.isArray(backupData.lists)).toBe(true);+ const exportedList = backupData.lists.find(+ (l: { name: string }) => l.name === "Backup Test List",+ );+ expect(exportedList).toBeDefined();+ expect(exportedList.icon).toBe("π");+ expect(exportedList.type).toBe("manual");+ expect(exportedList.id).toBe(listId);++ // Validate bookmark-to-list memberships+ const bm1 = backupData.bookmarks.find(+ (b: { title: string }) => b.title === "Test Bookmark 1",+ );+ const bm2 = backupData.bookmarks.find(+ (b: { title: string }) => b.title === "Test Bookmark 2",+ );+ const bm3 = backupData.bookmarks.find(+ (b: { title: string }) => b.title === "Test Bookmark 3",+ );+ expect(bm1.lists).toContain(listId);+ expect(bm2.lists).toContain(listId);+ expect(bm3.lists).toEqual([]);+// Step 7: Verify the backup appears in the list with updated statusconst { data: backupsData } = await client.GET("/backups");const listedBackup = backupsData!.backups.find((b) => b.id === backupId);diff --git a/packages/shared/import-export/exporters.ts b/packages/shared/import-export/exporters.tsindex 967937a4..f7a2581a 100644--- a/packages/shared/import-export/exporters.ts+++ b/packages/shared/import-export/exporters.ts@@ -1,11 +1,23 @@import { z } from "zod";import { BookmarkTypes, ZBookmark } from "../types/bookmarks";+import { ZBookmarkList } from "../types/lists";++export const zExportListSchema = z.object({+ id: z.string(),+ name: z.string(),+ description: z.string().nullable(),+ icon: z.string(),+ type: z.enum(["manual", "smart"]),+ query: z.string().nullable(),+ parentId: z.string().nullable(),+});export const zExportBookmarkSchema = z.object({createdAt: z.number(),title: z.string().nullable(),tags: z.array(z.string()),+ lists: z.array(z.string()).optional().default([]),content: z.discriminatedUnion("type", [z.object({@@ -24,10 +36,12 @@ export const zExportBookmarkSchema = z.object({export const zExportSchema = z.object({bookmarks: z.array(zExportBookmarkSchema),+ lists: z.array(zExportListSchema).optional().default([]),});export function toExportFormat(bookmark: ZBookmark,+ listIds?: string[],): z.infer<typeof zExportBookmarkSchema> {let content = null;switch (bookmark.content.type) {@@ -55,12 +69,27 @@ export function toExportFormat(? (bookmark.content.title ?? null): null),tags: bookmark.tags.map((t) => t.name),+ lists: listIds ?? [],content,note: bookmark.note ?? null,archived: bookmark.archived,};}+export function toExportListFormat(+ list: ZBookmarkList,+): z.infer<typeof zExportListSchema> {+ return {+ id: list.id,+ name: list.name,+ description: list.description ?? null,+ icon: list.icon,+ type: list.type,+ query: list.query ?? null,+ parentId: list.parentId,+ };+}+export function toNetscapeFormat(bookmarks: ZBookmark[]): string {const header = `<!DOCTYPE NETSCAPE-Bookmark-file-1><!-- This is an automatically generated file.diff --git a/packages/shared/import-export/importer.test.ts b/packages/shared/import-export/importer.test.tsindex f097f8d5..ff15953b 100644--- a/packages/shared/import-export/importer.test.ts+++ b/packages/shared/import-export/importer.test.ts@@ -10,45 +10,51 @@ const fakeFile = {describe("importBookmarksFromFile", () => {it("creates root list, folders and stages bookmarks with progress", async () => {const parsers = {- pocket: vi.fn().mockReturnValue([- {- title: "GitHub Repository",- content: { type: "link", url: "https://github.com/example/repo" },- tags: ["dev", "github"],- addDate: 100,- paths: [["Development", "Projects"]],- },- {- title: "My Notes",- content: { type: "text", text: "Important notes about the project" },- tags: ["notes"],- addDate: 200,- paths: [["Personal"]],- notes: "Additional context",- archived: true,- },- {- title: "Blog Post",- content: { type: "link", url: "https://example.com/blog" },- tags: ["reading", "tech"],- addDate: 300,- paths: [["Reading", "Tech"]],- },- {- title: "No Category Item",- content: { type: "link", url: "https://example.com/misc" },- tags: [],- addDate: 400,- paths: [],- },- {- title: "Duplicate URL Test",- content: { type: "link", url: "https://github.com/example/repo" },- tags: ["duplicate"],- addDate: 50, // Earlier date- paths: [["Development", "Duplicates"]],- },- ]),+ pocket: vi.fn().mockReturnValue({+ bookmarks: [+ {+ title: "GitHub Repository",+ content: { type: "link", url: "https://github.com/example/repo" },+ tags: ["dev", "github"],+ addDate: 100,+ paths: [["Development", "Projects"]],+ },+ {+ title: "My Notes",+ content: {+ type: "text",+ text: "Important notes about the project",+ },+ tags: ["notes"],+ addDate: 200,+ paths: [["Personal"]],+ notes: "Additional context",+ archived: true,+ },+ {+ title: "Blog Post",+ content: { type: "link", url: "https://example.com/blog" },+ tags: ["reading", "tech"],+ addDate: 300,+ paths: [["Reading", "Tech"]],+ },+ {+ title: "No Category Item",+ content: { type: "link", url: "https://example.com/misc" },+ tags: [],+ addDate: 400,+ paths: [],+ },+ {+ title: "Duplicate URL Test",+ content: { type: "link", url: "https://github.com/example/repo" },+ tags: ["duplicate"],+ addDate: 50, // Earlier date+ paths: [["Development", "Duplicates"]],+ },+ ],+ lists: [],+ }),};const createdLists: { name: string; icon: string; parentId?: string }[] =@@ -157,7 +163,9 @@ describe("importBookmarksFromFile", () => {});it("returns zero counts and null rootListId when no bookmarks", async () => {- const parsers = { html: vi.fn().mockReturnValue([]) };+ const parsers = {+ html: vi.fn().mockReturnValue({ bookmarks: [], lists: [] }),+ };const res = await importBookmarksFromFile({file: fakeFile,@@ -181,29 +189,32 @@ describe("importBookmarksFromFile", () => {it("stages all bookmarks successfully", async () => {const parsers = {- pocket: vi.fn().mockReturnValue([- {- title: "Bookmark 1",- content: { type: "link", url: "https://example.com/1" },- tags: ["tag1"],- addDate: 100,- paths: [["Category1"]],- },- {- title: "Bookmark 2",- content: { type: "link", url: "https://example.com/2" },- tags: ["tag2"],- addDate: 200,- paths: [["Category2"]],- },- {- title: "Bookmark 3",- content: { type: "link", url: "https://example.com/3" },- tags: ["tag3"],- addDate: 300,- paths: [["Category1"]],- },- ]),+ pocket: vi.fn().mockReturnValue({+ bookmarks: [+ {+ title: "Bookmark 1",+ content: { type: "link", url: "https://example.com/1" },+ tags: ["tag1"],+ addDate: 100,+ paths: [["Category1"]],+ },+ {+ title: "Bookmark 2",+ content: { type: "link", url: "https://example.com/2" },+ tags: ["tag2"],+ addDate: 200,+ paths: [["Category2"]],+ },+ {+ title: "Bookmark 3",+ content: { type: "link", url: "https://example.com/3" },+ tags: ["tag3"],+ addDate: 300,+ paths: [["Category1"]],+ },+ ],+ lists: [],+ }),};const createdLists: { name: string; icon: string; parentId?: string }[] =@@ -281,29 +292,32 @@ describe("importBookmarksFromFile", () => {it("stages bookmarks with different paths", async () => {const parsers = {- pocket: vi.fn().mockReturnValue([- {- title: "Bookmark 1",- content: { type: "link", url: "https://example.com/1" },- tags: ["tag1"],- addDate: 100,- paths: [["Path1"]],- },- {- title: "Bookmark 2",- content: { type: "link", url: "https://example.com/2" },- tags: ["tag2"],- addDate: 200,- paths: [["Path2"]],- },- {- title: "Bookmark 3",- content: { type: "link", url: "https://example.com/3" },- tags: ["tag3"],- addDate: 300,- paths: [["Path2"]],- },- ]),+ pocket: vi.fn().mockReturnValue({+ bookmarks: [+ {+ title: "Bookmark 1",+ content: { type: "link", url: "https://example.com/1" },+ tags: ["tag1"],+ addDate: 100,+ paths: [["Path1"]],+ },+ {+ title: "Bookmark 2",+ content: { type: "link", url: "https://example.com/2" },+ tags: ["tag2"],+ addDate: 200,+ paths: [["Path2"]],+ },+ {+ title: "Bookmark 3",+ content: { type: "link", url: "https://example.com/3" },+ tags: ["tag3"],+ addDate: 300,+ paths: [["Path2"]],+ },+ ],+ lists: [],+ }),};const createList = vi.fn(@@ -365,6 +379,205 @@ describe("importBookmarksFromFile", () => {expect(finalizeImportStaging).toHaveBeenCalledWith("session-1");});+ it("preserves separate list memberships when external list IDs differ", async () => {+ const parsers = {+ pocket: vi.fn().mockReturnValue({+ bookmarks: [+ {+ title: "Bookmark 1",+ content: { type: "link", url: "https://example.com/1" },+ tags: [],+ addDate: 100,+ paths: [],+ listExternalIds: ["child-1-id"],+ },+ {+ title: "Bookmark 2",+ content: { type: "link", url: "https://example.com/2" },+ tags: [],+ addDate: 200,+ paths: [],+ listExternalIds: ["child-2-id"],+ },+ ],+ lists: [+ {+ externalId: "parent-id",+ name: "Projects",+ parentExternalId: null,+ type: "manual",+ },+ {+ externalId: "child-1-id",+ name: "Inbox",+ parentExternalId: "parent-id",+ type: "manual",+ },+ {+ externalId: "child-2-id",+ name: "Inbox",+ parentExternalId: "parent-id",+ type: "manual",+ },+ ],+ }),+ };++ let idCounter = 0;+ const createdLists: { id: string; name: string; parentId?: string }[] = [];+ const createList = vi.fn(+ async (input: { name: string; icon: string; parentId?: string }) => {+ const id = `list-${idCounter++}`;+ createdLists.push({ id, name: input.name, parentId: input.parentId });+ return { id };+ },+ );++ const stagedBookmarks: StagedBookmark[] = [];+ const stageImportedBookmarks = vi.fn(+ async (input: {+ importSessionId: string;+ bookmarks: StagedBookmark[];+ }) => {+ stagedBookmarks.push(...input.bookmarks);+ },+ );++ await importBookmarksFromFile(+ {+ file: fakeFile,+ source: "pocket",+ rootListName: "Imported",+ deps: {+ createList,+ stageImportedBookmarks,+ finalizeImportStaging: vi.fn(),+ createImportSession: vi.fn(async () => ({ id: "session-1" })),+ },+ },+ { parsers },+ );++ const projectsFolder = createdLists.find(+ (list) => list.name === "Projects",+ );+ expect(projectsFolder).toBeDefined();++ const duplicateFolders = createdLists.filter(+ (list) => list.name === "Inbox" && list.parentId === projectsFolder?.id,+ );+ expect(duplicateFolders).toHaveLength(2);++ const firstBookmark = stagedBookmarks.find(+ (bookmark) => bookmark.url === "https://example.com/1",+ );+ const secondBookmark = stagedBookmarks.find(+ (bookmark) => bookmark.url === "https://example.com/2",+ );++ expect(firstBookmark).toBeDefined();+ expect(secondBookmark).toBeDefined();+ expect(firstBookmark?.listIds[0]).not.toEqual(secondBookmark?.listIds[0]);+ expect(duplicateFolders.map((list) => list.id)).toContain(+ firstBookmark?.listIds[0],+ );+ expect(duplicateFolders.map((list) => list.id)).toContain(+ secondBookmark?.listIds[0],+ );+ });++ it("creates smart lists with their queries during import", async () => {+ const parsers = {+ karakeep: vi.fn().mockReturnValue({+ bookmarks: [+ {+ title: "Bookmark 1",+ content: { type: "link", url: "https://example.com/1" },+ tags: [],+ addDate: 100,+ paths: [],+ listExternalIds: ["manual-list-id"],+ },+ ],+ lists: [+ {+ externalId: "manual-list-id",+ name: "Manual",+ icon: "β",+ description: "Manual list description",+ parentExternalId: null,+ type: "manual",+ },+ {+ externalId: "smart-list-id",+ name: "Smart",+ icon: "β‘",+ description: "Smart list description",+ parentExternalId: null,+ type: "smart",+ query: "tag:read-later",+ },+ ],+ }),+ };++ const createdLists: {+ name: string;+ icon: string;+ description?: string;+ parentId?: string;+ type?: "manual" | "smart";+ query?: string;+ }[] = [];++ const createList = vi.fn(+ async (input: {+ name: string;+ icon: string;+ description?: string;+ parentId?: string;+ type?: "manual" | "smart";+ query?: string;+ }) => {+ createdLists.push(input);+ return {+ id: `${input.parentId ? input.parentId + "/" : ""}${input.name}`,+ };+ },+ );++ await importBookmarksFromFile(+ {+ file: fakeFile,+ source: "karakeep",+ rootListName: "Imported",+ deps: {+ createList,+ stageImportedBookmarks: vi.fn(async () => undefined),+ finalizeImportStaging: vi.fn(),+ createImportSession: vi.fn(async () => ({ id: "session-1" })),+ },+ },+ { parsers },+ );++ expect(createdLists).toContainEqual({+ name: "Smart",+ parentId: "Imported",+ icon: "β‘",+ description: "Smart list description",+ type: "smart",+ query: "tag:read-later",+ });++ expect(createdLists).toContainEqual({+ name: "Manual",+ parentId: "Imported",+ icon: "β",+ description: "Manual list description",+ });+ });+it("handles HTML bookmarks with empty folder names", async () => {const htmlContent = `<!DOCTYPE NETSCAPE-Bookmark-file-1><META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">diff --git a/packages/shared/import-export/importer.ts b/packages/shared/import-export/importer.tsindex be24ca73..a629d39c 100644--- a/packages/shared/import-export/importer.ts+++ b/packages/shared/import-export/importer.ts@@ -1,5 +1,5 @@import { MAX_LIST_NAME_LENGTH } from "../types/lists";-import { ImportSource, ParsedBookmark, parseImportFile } from "./parsers";+import { ImportSource, ParsedImportFile, parseImportFile } from "./parsers";export interface ImportCounts {successes: number;@@ -23,7 +23,10 @@ export interface ImportDeps {createList: (input: {name: string;icon: string;+ description?: string;parentId?: string;+ type?: "manual" | "smart";+ query?: string;}) => Promise<{ id: string }>;stageImportedBookmarks: (input: {importSessionId: string;@@ -39,7 +42,7 @@ export interface ImportDeps {export interface ImportOptions {concurrencyLimit?: number;parsers?: Partial<- Record<ImportSource, (textContent: string) => ParsedBookmark[]>+ Record<ImportSource, (textContent: string) => ParsedImportFile>>;}@@ -68,9 +71,11 @@ export async function importBookmarksFromFile(const { parsers } = options;const textContent = await file.text();- const parsedBookmarks = parsers?.[source]+ const parsedImport = parsers?.[source]? parsers[source]!(textContent): parseImportFile(source, textContent);+ const parsedBookmarks = parsedImport.bookmarks;+ const parsedLists = parsedImport.lists;if (parsedBookmarks.length === 0) {return {counts: { successes: 0, failures: 0, alreadyExisted: 0, total: 0 },@@ -87,34 +92,100 @@ export async function importBookmarksFromFile(onProgress?.(0, parsedBookmarks.length);+ const externalListIdToCreatedListId: Record<string, string> = {};+ if (parsedLists.length > 0) {+ const unresolvedLists = new Map(+ parsedLists.map((list) => [list.externalId, list]),+ );++ while (unresolvedLists.size > 0) {+ let createdAny = false;++ for (const [externalId, list] of unresolvedLists) {+ if (+ list.parentExternalId &&+ !externalListIdToCreatedListId[list.parentExternalId]+ ) {+ continue;+ }++ const parentId = list.parentExternalId+ ? externalListIdToCreatedListId[list.parentExternalId]+ : rootList.id;++ const createdList = await deps.createList({+ name: list.name.substring(0, MAX_LIST_NAME_LENGTH),+ parentId,+ icon: list.icon ?? "π",+ description: list.description,+ ...(list.type === "smart" && list.query+ ? { type: "smart", query: list.query }+ : {}),+ });+ externalListIdToCreatedListId[externalId] = createdList.id;+ unresolvedLists.delete(externalId);+ createdAny = true;+ }++ // Break cycles or unresolved parent references by attaching remaining+ // lists to the import root.+ if (!createdAny) {+ for (const [externalId, list] of unresolvedLists) {+ const createdList = await deps.createList({+ name: list.name.substring(0, MAX_LIST_NAME_LENGTH),+ parentId: rootList.id,+ icon: list.icon ?? "π",+ description: list.description,+ ...(list.type === "smart" && list.query+ ? { type: "smart", query: list.query }+ : {}),+ });+ externalListIdToCreatedListId[externalId] = createdList.id;+ }+ unresolvedLists.clear();+ }+ }+ }+const PATH_DELIMITER = "$$__$$";+ const getPathKey = (parts: string[]) => parts.join(PATH_DELIMITER);+ const bookmarksWithPathMembership = parsedBookmarks.filter(+ (bookmark) =>+ !bookmark.listExternalIds || bookmark.listExternalIds.length === 0,+ );// Build required paths- const allRequiredPaths = new Set<string>();- for (const bookmark of parsedBookmarks) {+ const allRequiredPaths = new Map<string, string>();+ for (const bookmark of bookmarksWithPathMembership) {for (const path of bookmark.paths) {if (path && path.length > 0) {for (let i = 1; i <= path.length; i++) {const subPath = path.slice(0, i);- const pathKey = subPath.join(PATH_DELIMITER);- allRequiredPaths.add(pathKey);+ const pathKey = getPathKey(subPath);+ const folderName = subPath[subPath.length - 1];+ if (!allRequiredPaths.has(pathKey)) {+ allRequiredPaths.set(pathKey, folderName);+ }}}}}- const allRequiredPathsArray = Array.from(allRequiredPaths).sort(- (a, b) => a.split(PATH_DELIMITER).length - b.split(PATH_DELIMITER).length,- );+ const allRequiredPathsArray = Array.from(allRequiredPaths.entries())+ .map(([pathKey, folderName]) => ({ pathKey, folderName }))+ .sort(+ (a, b) =>+ a.pathKey.split(PATH_DELIMITER).length -+ b.pathKey.split(PATH_DELIMITER).length,+ );const pathMap: Record<string, string> = { "": rootList.id };- for (const pathKey of allRequiredPathsArray) {+ for (const { pathKey, folderName } of allRequiredPathsArray) {const parts = pathKey.split(PATH_DELIMITER);const parentKey = parts.slice(0, -1).join(PATH_DELIMITER);const parentId = pathMap[parentKey] || rootList.id;- const folderName = parts[parts.length - 1];const folderList = await deps.createList({name: folderName.substring(0, MAX_LIST_NAME_LENGTH),parentId,@@ -127,7 +198,7 @@ export async function importBookmarksFromFile(const bookmarksToStage: StagedBookmark[] = parsedBookmarks.map((bookmark) => {// Convert paths to list IDs using pathMap// If no paths, assign to root list- const listIds =+ const listIdsFromPaths =bookmark.paths.length === 0? [rootList.id]: bookmark.paths@@ -135,11 +206,26 @@ export async function importBookmarksFromFile(if (path.length === 0) {return rootList.id;}- const pathKey = path.join(PATH_DELIMITER);+ const pathKey = getPathKey(path);return pathMap[pathKey] || rootList.id;}).filter((id, index, arr) => arr.indexOf(id) === index); // dedupe+ const externalListIds = bookmark.listExternalIds ?? [];+ const listIdsFromExternalListIds =+ externalListIds.length > 0+ ? [+ ...new Set(+ externalListIds.map((id) => externalListIdToCreatedListId[id]),+ ),+ ].filter((id): id is string => Boolean(id))+ : [];++ const listIds =+ listIdsFromExternalListIds.length > 0+ ? listIdsFromExternalListIds+ : listIdsFromPaths;+// Determine type and extract content appropriatelylet type: "link" | "text" | "asset" = "link";let url: string | undefined;diff --git a/packages/shared/import-export/index.ts b/packages/shared/import-export/index.tsindex dd266e06..a6940306 100644--- a/packages/shared/import-export/index.ts+++ b/packages/shared/import-export/index.ts@@ -1,4 +1,9 @@export * from "./exporters";export * from "./importer";-export type { ImportSource, ParsedBookmark } from "./parsers";+export type {+ ImportSource,+ ParsedBookmark,+ ParsedImportFile,+ ParsedImportList,+} from "./parsers";export { parseImportFile } from "./parsers";diff --git a/packages/shared/import-export/parsers.test.ts b/packages/shared/import-export/parsers.test.tsindex 18502305..a35d7bb8 100644--- a/packages/shared/import-export/parsers.test.ts+++ b/packages/shared/import-export/parsers.test.ts@@ -12,7 +12,9 @@ describe("parseNetscapeBookmarkFile", () => {<DT><A HREF="https://example.com" ADD_DATE="1234567890">Example Site</A></DL><p>`;- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(1);expect(result[0]).toMatchObject({@@ -36,7 +38,9 @@ describe("parseNetscapeBookmarkFile", () => {<DT><A HREF="https://example.com" ADD_DATE="1234567890" TAGS="tag1,tag2,tag3">Example Site</A></DL><p>`;- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(1);expect(result[0].tags).toEqual(["tag1", "tag2", "tag3"]);@@ -57,7 +61,9 @@ describe("parseNetscapeBookmarkFile", () => {</DL><p></DL><p>`;- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(1);expect(result[0]).toMatchObject({@@ -85,7 +91,9 @@ describe("parseNetscapeBookmarkFile", () => {</DL><p></DL><p>`;- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(1);expect(result[0].paths).toEqual([["Named Folder", "Unnamed"]]);@@ -108,7 +116,9 @@ describe("parseNetscapeBookmarkFile", () => {</DL><p></DL><p>`;- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(3);@@ -141,7 +151,9 @@ describe("parseNetscapeBookmarkFile", () => {<DT><A HREF="https://example2.com" ADD_DATE="1234567891">Bookmark 2</A></DL><p>`;- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(2);expect(result[0].paths).toEqual([[]]);@@ -169,7 +181,9 @@ describe("parseNetscapeBookmarkFile", () => {</DL><p></DL><p>`;- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(1);expect(result[0].paths).toEqual([["Level1", "Level2", "Level3", "Level4"]]);@@ -191,7 +205,9 @@ describe("parseNetscapeBookmarkFile", () => {</DL><p></DL><p>`;- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(1);expect(result[0]).toMatchObject({@@ -218,7 +234,9 @@ describe("parseNetscapeBookmarkFile", () => {// Note: The current parser doesn't extract DD notes, but this test// documents the expected behavior if/when DD parsing is added- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(1);expect(result[0].content).toMatchObject({@@ -236,7 +254,9 @@ describe("parseNetscapeBookmarkFile", () => {<DT><A HREF="https://example.com">No Date Bookmark</A></DL><p>`;- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(1);expect(result[0].addDate).toBeUndefined();@@ -251,7 +271,9 @@ describe("parseNetscapeBookmarkFile", () => {<DT><A ADD_DATE="1234567890">No URL Bookmark</A></DL><p>`;- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(1);expect(result[0].content).toBeUndefined();@@ -271,7 +293,9 @@ describe("parseNetscapeBookmarkFile", () => {<DT><A HREF="https://root2.com" ADD_DATE="1234567892">Root Bookmark 2</A></DL><p>`;- const result = parseImportFile("html", html);+ const parsed = parseImportFile("html", html);++ const result = parsed.bookmarks;expect(result).toHaveLength(3);expect(result[0]).toMatchObject({@@ -299,3 +323,141 @@ describe("parseNetscapeBookmarkFile", () => {);});});++describe("parseKarakeepBookmarkFile", () => {+ it("keeps distinct identities for duplicate sibling list names", () => {+ const json = JSON.stringify({+ lists: [+ {+ id: "parent",+ name: "Projects",+ description: null,+ icon: "π",+ type: "manual",+ query: null,+ parentId: null,+ },+ {+ id: "child-1",+ name: "Inbox",+ description: null,+ icon: "π",+ type: "manual",+ query: null,+ parentId: "parent",+ },+ {+ id: "child-2",+ name: "Inbox",+ description: null,+ icon: "π",+ type: "manual",+ query: null,+ parentId: "parent",+ },+ ],+ bookmarks: [+ {+ createdAt: 123,+ title: "One",+ tags: [],+ lists: ["child-1"],+ content: { type: "link", url: "https://one.example" },+ note: null,+ archived: false,+ },+ {+ createdAt: 456,+ title: "Two",+ tags: [],+ lists: ["child-2"],+ content: { type: "link", url: "https://two.example" },+ note: null,+ archived: false,+ },+ ],+ });++ const result = parseImportFile("karakeep", json);++ expect(result.lists).toEqual([+ {+ externalId: "parent",+ name: "Projects",+ icon: "π",+ description: undefined,+ parentExternalId: null,+ type: "manual",+ },+ {+ externalId: "child-1",+ name: "Inbox",+ icon: "π",+ description: undefined,+ parentExternalId: "parent",+ type: "manual",+ },+ {+ externalId: "child-2",+ name: "Inbox",+ icon: "π",+ description: undefined,+ parentExternalId: "parent",+ type: "manual",+ },+ ]);++ expect(result.bookmarks).toHaveLength(2);+ expect(result.bookmarks[0].listExternalIds).toEqual(["child-1"]);+ expect(result.bookmarks[1].listExternalIds).toEqual(["child-2"]);+ });++ it("preserves smart list query metadata", () => {+ const json = JSON.stringify({+ lists: [+ {+ id: "manual",+ name: "Manual",+ description: null,+ icon: "π",+ type: "manual",+ query: null,+ parentId: null,+ },+ {+ id: "smart",+ name: "Smart",+ description: "Smart list description",+ icon: "β‘",+ type: "smart",+ query: "tag:read-later",+ parentId: null,+ },+ ],+ bookmarks: [+ {+ createdAt: 123,+ title: "One",+ tags: [],+ lists: ["manual", "smart"],+ content: { type: "link", url: "https://one.example" },+ note: null,+ archived: false,+ },+ ],+ });++ const result = parseImportFile("karakeep", json);++ expect(result.lists).toContainEqual({+ externalId: "smart",+ name: "Smart",+ icon: "β‘",+ description: "Smart list description",+ parentExternalId: null,+ type: "smart",+ query: "tag:read-later",+ });+ expect(result.bookmarks[0].listExternalIds).toEqual(["manual"]);+ });+});diff --git a/packages/shared/import-export/parsers.ts b/packages/shared/import-export/parsers.tsindex 24d85c80..b46d0e65 100644--- a/packages/shared/import-export/parsers.ts+++ b/packages/shared/import-export/parsers.ts@@ -29,6 +29,23 @@ export interface ParsedBookmark {notes?: string;archived?: boolean;paths: string[][];+ // Optional list IDs from the source file (used with top-level `lists`).+ listExternalIds?: string[];+}++export interface ParsedImportList {+ externalId: string;+ name: string;+ icon?: string;+ description?: string;+ parentExternalId: string | null;+ type: "manual" | "smart";+ query?: string;+}++export interface ParsedImportFile {+ bookmarks: ParsedBookmark[];+ lists: ParsedImportList[];}function parseNetscapeBookmarkFile(textContent: string): ParsedBookmark[] {@@ -161,7 +178,7 @@ function parseMatterBookmarkFile(textContent: string): ParsedBookmark[] {});}-function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] {+function parseKarakeepBookmarkFile(textContent: string): ParsedImportFile {const parsed = zExportSchema.safeParse(JSON.parse(textContent));if (!parsed.success) {throw new Error(@@ -169,7 +186,22 @@ function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] {);}- return parsed.data.bookmarks.map((bookmark) => {+ const exportedLists = parsed.data.lists ?? [];+ const parsedLists: ParsedImportList[] = exportedLists.map((list) => ({+ externalId: list.id,+ name: list.name,+ icon: list.icon,+ description: list.description ?? undefined,+ parentExternalId: list.parentId,+ type: list.type,+ query: list.type === "smart" ? (list.query ?? undefined) : undefined,+ }));++ const manualListIds = new Set(+ exportedLists.filter((l) => l.type === "manual").map((l) => l.id),+ );++ const parsedBookmarks = parsed.data.bookmarks.map((bookmark) => {let content = undefined;if (bookmark.content?.type == BookmarkTypes.LINK) {content = {@@ -182,6 +214,7 @@ function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] {text: bookmark.content.text,};}+return {title: bookmark.title ?? "",content,@@ -189,9 +222,17 @@ function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] {addDate: bookmark.createdAt,notes: bookmark.note ?? undefined,archived: bookmark.archived,- paths: [], // TODO+ paths: [],+ listExternalIds: (bookmark.lists ?? []).filter((listId) =>+ manualListIds.has(listId),+ ),};});++ return {+ bookmarks: parsedBookmarks,+ lists: parsedLists,+ };}function parseOmnivoreBookmarkFile(textContent: string): ParsedBookmark[] {@@ -429,6 +470,14 @@ function deduplicateBookmarks(bookmarks: ParsedBookmark[]): ParsedBookmark[] {existing.tags = [...new Set([...existing.tags, ...bookmark.tags])];// Merge pathsexisting.paths = [...existing.paths, ...bookmark.paths];+ if (existing.listExternalIds || bookmark.listExternalIds) {+ existing.listExternalIds = [+ ...new Set([+ ...(existing.listExternalIds ?? []),+ ...(bookmark.listExternalIds ?? []),+ ]),+ ];+ }const existingDate = existing.addDate ?? Infinity;const newDate = bookmark.addDate ?? Infinity;if (newDate < existingDate) {@@ -460,7 +509,15 @@ function deduplicateBookmarks(bookmarks: ParsedBookmark[]): ParsedBookmark[] {export function parseImportFile(source: ImportSource,textContent: string,-): ParsedBookmark[] {+): ParsedImportFile {+ if (source === "karakeep") {+ const parsed = parseKarakeepBookmarkFile(textContent);+ return {+ bookmarks: deduplicateBookmarks(parsed.bookmarks),+ lists: parsed.lists,+ };+ }+let result: ParsedBookmark[];switch (source) {case "html":@@ -472,9 +529,6 @@ export function parseImportFile(case "matter":result = parseMatterBookmarkFile(textContent);break;- case "karakeep":- result = parseKarakeepBookmarkFile(textContent);- break;case "omnivore":result = parseOmnivoreBookmarkFile(textContent);break;@@ -491,5 +545,5 @@ export function parseImportFile(result = parseInstapaperBookmarkFile(textContent);break;}- return deduplicateBookmarks(result);+ return { bookmarks: deduplicateBookmarks(result), lists: [] };}