A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
chore: Add seed snapshot generator (#2829)
Mohamed Bassem53da5e2bbe94
diff --git a/.dockerignore b/.dockerignoreindex 748c949b..96954fb7 100644--- a/.dockerignore+++ b/.dockerignore@@ -19,6 +19,7 @@ apps/landingapps/browser-extensionpackages/e2e_testspackages/benchmarks+tools/seed-snapshot# Aider.aider*diff --git a/package.json b/package.jsonindex b0a5b7dc..c355d5c4 100644--- a/package.json+++ b/package.json@@ -11,6 +11,8 @@"db:generate": "pnpm --filter @karakeep/db run generate","db:migrate": "pnpm --filter @karakeep/db run migrate","db:studio": "pnpm --filter @karakeep/db studio",+ "seed:apply": "pnpm --filter @karakeep/seed-snapshot run apply",+ "seed:snapshot": "pnpm --filter @karakeep/seed-snapshot run snapshot","workers": "pnpm --filter @karakeep/workers run start","web": "pnpm --filter @karakeep/web run dev","web:portless": "pnpm --filter @karakeep/web run dev:portless",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 71c9a6f8..2e3d52c6 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1618,6 +1618,31 @@ importers:specifier: ^5.9version: 5.9.3+ tools/seed-snapshot:+ dependencies:+ '@karakeep/shared':+ specifier: workspace:^0.1.0+ version: link:../../packages/shared+ '@karakeep/trpc':+ specifier: workspace:^0.1.0+ version: link:../../packages/trpc+ '@trpc/client':+ specifier: ^11.9.0+ superjson:+ specifier: ^2.2.1+ version: 2.2.1+ devDependencies:+ '@karakeep/tsconfig':+ specifier: workspace:^0.1.0+ version: link:../../tooling/typescript+ oxlint:+ specifier: ^1.60.0+ version: 1.60.0+ tsx:+ specifier: ^4.8.1+ version: 4.21.0+packages:'@0no-co/[email protected]':diff --git a/snapshots/seed-data-2026-05-20-163735.json b/snapshots/seed-data-2026-05-20-163735.jsonnew file mode 100644index 00000000..cff6efe7--- /dev/null+++ b/snapshots/seed-data-2026-05-20-163735.json@@ -0,0 +1,256 @@+{+ "createdAt": "2026-05-20T15:37:35.334Z",+ "sourceCommit": "021cb12579bdf4cc5e2b50f09170c23f99ed6eab",+ "archive": "seed-data-2026-05-20-163735.tar.gz",+ "users": [+ {+ "email": "[email protected]",+ "password": "test1234",+ "seeded": true,+ "apiKeyName": "[email protected]",+ "counts": {+ "bookmarks": 20,+ "tags": 3,+ "manualLists": 2,+ "smartLists": 2+ },+ "bookmarks": [+ {+ "id": "km2jba5wyxzsa07ea1rvlkw6",+ "url": "https://www.nasa.gov/",+ "title": "Seed snapshot bookmark 1",+ "crawlStatus": "success"+ },+ {+ "id": "kxriupr76n9swxb1c5ypzk7x",+ "url": "https://www.nationalgeographic.com/",+ "title": "Seed snapshot bookmark 2",+ "crawlStatus": "success"+ },+ {+ "id": "smw5grpmamknt52h2egre7w9",+ "url": "https://www.theverge.com/",+ "title": "Seed snapshot bookmark 3",+ "crawlStatus": "success"+ },+ {+ "id": "t47py0t3mi1abngrzbh0ve67",+ "url": "https://www.wired.com/",+ "title": "Seed snapshot bookmark 4",+ "crawlStatus": "success"+ },+ {+ "id": "exe0kj1nr1yheba71hmhnpr7",+ "url": "https://www.apple.com/ipad-pro/",+ "title": "Seed snapshot bookmark 5",+ "crawlStatus": "success"+ },+ {+ "id": "yx72o5hos53bmbjhoxcv9z2g",+ "url": "https://www.figma.com/",+ "title": "Seed snapshot bookmark 6",+ "crawlStatus": "success"+ },+ {+ "id": "xkoqt13nxufsk5b1y60fxo5k",+ "url": "https://linear.app/",+ "title": "Seed snapshot bookmark 7",+ "crawlStatus": "success"+ },+ {+ "id": "thv2c1k4z4reuu7yc6482i8t",+ "url": "https://www.notion.com/",+ "title": "Seed snapshot bookmark 8",+ "crawlStatus": "success"+ },+ {+ "id": "psg7m6mhwm1xfnpyqskb6jy5",+ "url": "https://stripe.com/",+ "title": "Seed snapshot bookmark 9",+ "crawlStatus": "success"+ },+ {+ "id": "gmab7mbludxywqxx8hz9qxt6",+ "url": "https://www.shopify.com/",+ "title": "Seed snapshot bookmark 10",+ "crawlStatus": "success"+ },+ {+ "id": "e2ej14zclzd0gw91u17fwux9",+ "url": "https://developer.mozilla.org/en-US/docs/Web",+ "title": "Seed snapshot bookmark 11",+ "crawlStatus": "success"+ },+ {+ "id": "j0b03m8ks0cmgo8lkokku3mn",+ "url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript",+ "title": "Seed snapshot bookmark 12",+ "crawlStatus": "success"+ },+ {+ "id": "aflpy1knahedicy7hihiwz0k",+ "url": "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API",+ "title": "Seed snapshot bookmark 13",+ "crawlStatus": "success"+ },+ {+ "id": "lc8sdxnnbh6lxrp79ou39fnb",+ "url": "https://developer.mozilla.org/en-US/docs/Web/CSS",+ "title": "Seed snapshot bookmark 14",+ "crawlStatus": "success"+ },+ {+ "id": "sb4rtshzmsa3sasllcn54gnz",+ "url": "https://web.dev/learn/html",+ "title": "Seed snapshot bookmark 15",+ "crawlStatus": "success"+ },+ {+ "id": "pk4h0d754vjs6dm3508c5q78",+ "url": "https://web.dev/learn/css",+ "title": "Seed snapshot bookmark 16",+ "crawlStatus": "success"+ },+ {+ "id": "jqdfmk4yb9ekec4kkg8a69cg",+ "url": "https://web.dev/learn/javascript",+ "title": "Seed snapshot bookmark 17",+ "crawlStatus": "success"+ },+ {+ "id": "zcbcwbus69sc701khrtmyccz",+ "url": "https://www.raspberrypi.com/products/raspberry-pi-5/",+ "title": "Seed snapshot bookmark 18",+ "crawlStatus": "success"+ },+ {+ "id": "uhvuf05w0ldy0k3bqbcslofu",+ "url": "https://www.postgresql.org/docs/current/index.html",+ "title": "Seed snapshot bookmark 19",+ "crawlStatus": "success"+ },+ {+ "id": "q5mf9byhs1nekffbs25ty8rs",+ "url": "https://docs.docker.com/get-started/",+ "title": "Seed snapshot bookmark 20",+ "crawlStatus": "success"+ }+ ],+ "tags": [+ "research",+ "docs",+ "product"+ ],+ "lists": [+ {+ "name": "Reading Queue",+ "type": "manual",+ "parentName": null,+ "query": null+ },+ {+ "name": "Reference",+ "type": "manual",+ "parentName": "Reading Queue",+ "query": null+ },+ {+ "name": "Research Smart",+ "type": "smart",+ "parentName": null,+ "query": "tag:research"+ },+ {+ "name": "Favorites Smart",+ "type": "smart",+ "parentName": "Reading Queue",+ "query": "is:fav"+ }+ ]+ },+ {+ "email": "[email protected]",+ "password": "test1234",+ "seeded": true,+ "apiKeyName": "[email protected]",+ "counts": {+ "bookmarks": 4,+ "tags": 3,+ "manualLists": 2,+ "smartLists": 2+ },+ "bookmarks": [+ {+ "id": "c9ybiun3c0lwfqvbzuumasvq",+ "url": "https://github.com/features/actions",+ "title": "Seed snapshot bookmark 1",+ "crawlStatus": "success"+ },+ {+ "id": "z09y6r8bmn0c9j0z687esgus",+ "url": "https://www.cloudflare.com/",+ "title": "Seed snapshot bookmark 2",+ "crawlStatus": "success"+ },+ {+ "id": "k9bkj37wdby9yxn726fsbc4f",+ "url": "https://web.dev/",+ "title": "Seed snapshot bookmark 3",+ "crawlStatus": "success"+ },+ {+ "id": "tvsyx6oswqu8loayf4gc7i7y",+ "url": "https://www.raspberrypi.com/products/raspberry-pi-5/",+ "title": "Seed snapshot bookmark 4",+ "crawlStatus": "success"+ }+ ],+ "tags": [+ "engineering",+ "design",+ "ops"+ ],+ "lists": [+ {+ "name": "Workbench",+ "type": "manual",+ "parentName": null,+ "query": null+ },+ {+ "name": "Deep Dives",+ "type": "manual",+ "parentName": "Workbench",+ "query": null+ },+ {+ "name": "Engineering Smart",+ "type": "smart",+ "parentName": null,+ "query": "tag:engineering"+ },+ {+ "name": "Archived Smart",+ "type": "smart",+ "parentName": "Workbench",+ "query": "is:archived"+ }+ ]+ },+ {+ "email": "[email protected]",+ "password": "test1234",+ "seeded": false,+ "apiKeyName": "[email protected]",+ "counts": {+ "bookmarks": 0,+ "tags": 0,+ "manualLists": 0,+ "smartLists": 0+ },+ "bookmarks": [],+ "tags": [],+ "lists": []+ }+ ]+}diff --git a/snapshots/seed-data-2026-05-20-163735.tar.gz b/snapshots/seed-data-2026-05-20-163735.tar.gznew file mode 100644index 00000000..5c42603fBinary files /dev/null and b/snapshots/seed-data-2026-05-20-163735.tar.gz differdiff --git a/tools/seed-snapshot/.gitignore b/tools/seed-snapshot/.gitignorenew file mode 100644index 00000000..5f65e0ba--- /dev/null+++ b/tools/seed-snapshot/.gitignore@@ -0,0 +1,2 @@+.tmp/+docker-logs/diff --git a/tools/seed-snapshot/docker-compose.yml b/tools/seed-snapshot/docker-compose.ymlnew file mode 100644index 00000000..98697924--- /dev/null+++ b/tools/seed-snapshot/docker-compose.yml@@ -0,0 +1,35 @@+services:+ web:+ build:+ dockerfile: docker/Dockerfile+ context: ../../+ target: aio+ restart: unless-stopped+ ports:+ - "${KARAKEEP_PORT:-3000}:3000"+ environment:+ DATA_DIR: /data+ NEXTAUTH_SECRET: seed-snapshot-secret+ NEXTAUTH_URL: http://localhost:${KARAKEEP_PORT:-3000}+ MEILI_MASTER_KEY: dummy+ MEILI_ADDR: http://meilisearch:7700+ BROWSER_WEB_URL: http://chrome:9222+ CRAWLER_NUM_WORKERS: 4+ volumes:+ - "${SEED_SNAPSHOT_DATA_DIR:?SEED_SNAPSHOT_DATA_DIR is required}:/data"+ meilisearch:+ image: getmeili/meilisearch:v1.41.0+ restart: unless-stopped+ environment:+ MEILI_NO_ANALYTICS: "true"+ MEILI_MASTER_KEY: dummy+ chrome:+ image: gcr.io/zenika-hub/alpine-chrome:124+ restart: unless-stopped+ command:+ - --no-sandbox+ - --disable-gpu+ - --disable-dev-shm-usage+ - --remote-debugging-address=0.0.0.0+ - --remote-debugging-port=9222+ - --hide-scrollbarsdiff --git a/tools/seed-snapshot/package.json b/tools/seed-snapshot/package.jsonnew file mode 100644index 00000000..b80f31e2--- /dev/null+++ b/tools/seed-snapshot/package.json@@ -0,0 +1,27 @@+{+ "$schema": "https://json.schemastore.org/package.json",+ "name": "@karakeep/seed-snapshot",+ "version": "0.1.0",+ "private": true,+ "type": "module",+ "scripts": {+ "apply": "tsx src/apply.ts",+ "snapshot": "tsx src/index.ts",+ "lint": "oxlint --ignore-path=../../.gitignore .",+ "lint:fix": "oxlint --ignore-path=../../.gitignore . --fix",+ "format": "oxfmt --ignore-path=../../.gitignore --check .",+ "format:fix": "oxfmt --ignore-path=../../.gitignore .",+ "typecheck": "tsc --noEmit"+ },+ "dependencies": {+ "@karakeep/shared": "workspace:^0.1.0",+ "@karakeep/trpc": "workspace:^0.1.0",+ "@trpc/client": "^11.9.0",+ "superjson": "^2.2.1"+ },+ "devDependencies": {+ "@karakeep/tsconfig": "workspace:^0.1.0",+ "oxlint": "^1.60.0",+ "tsx": "^4.8.1"+ }+}diff --git a/tools/seed-snapshot/src/apply.ts b/tools/seed-snapshot/src/apply.tsnew file mode 100644index 00000000..f68e7f58--- /dev/null+++ b/tools/seed-snapshot/src/apply.ts@@ -0,0 +1,90 @@+import { execFileSync } from "child_process";+import fs from "fs/promises";+import path from "path";+import { fileURLToPath } from "url";++const SNAPSHOT_PATTERN = /^seed-data-\d{4}-\d{2}-\d{2}-\d{6}\.tar\.gz$/;++function logStep(title: string): void {+ console.log(`\n== ${title}`);+}++function logInfo(message: string): void {+ console.log(` -- ${message}`);+}++function logSuccess(message: string): void {+ console.log(` OK ${message}`);+}++function hasForceFlag(): boolean {+ return process.argv.slice(2).includes("--force");+}++async function pathExists(targetPath: string): Promise<boolean> {+ try {+ await fs.stat(targetPath);+ return true;+ } catch (error) {+ if (+ error &&+ typeof error === "object" &&+ "code" in error &&+ error.code === "ENOENT"+ ) {+ return false;+ }+ throw error;+ }+}++async function findLatestSnapshot(snapshotsDir: string): Promise<string> {+ const entries = await fs.readdir(snapshotsDir);+ const snapshots = entries+ .filter((entry) => SNAPSHOT_PATTERN.test(entry))+ .sort();++ const latest = snapshots.at(-1);+ if (!latest) {+ throw new Error(`No seed snapshots found in ${snapshotsDir}`);+ }++ return path.join(snapshotsDir, latest);+}++async function main(): Promise<void> {+ const __dirname = path.dirname(fileURLToPath(import.meta.url));+ const repoRoot = path.join(__dirname, "..", "..", "..");+ const snapshotsDir = path.join(repoRoot, "snapshots");+ const dataDir = path.join(repoRoot, "data");+ const force = hasForceFlag();++ logStep("Finding latest seed snapshot");+ const snapshotPath = await findLatestSnapshot(snapshotsDir);+ logInfo(`Using ${path.relative(repoRoot, snapshotPath)}`);++ if (await pathExists(dataDir)) {+ if (!force) {+ throw new Error(+ `Refusing to overwrite existing data dir at ${dataDir}. Re-run with --force to replace it.`,+ );+ }++ logStep("Removing existing data dir");+ await fs.rm(dataDir, { force: true, recursive: true });+ }++ logStep("Applying seed snapshot");+ await fs.mkdir(dataDir, { recursive: true });+ execFileSync("tar", ["-xzf", snapshotPath, "-C", dataDir], {+ cwd: repoRoot,+ stdio: "inherit",+ });++ logSuccess(`Applied snapshot to ${dataDir}`);+}++main().catch((error) => {+ console.error(error);+ process.exitCode = 1;+});diff --git a/tools/seed-snapshot/src/index.ts b/tools/seed-snapshot/src/index.tsnew file mode 100644index 00000000..f031a8d5--- /dev/null+++ b/tools/seed-snapshot/src/index.ts@@ -0,0 +1,623 @@+import { execFileSync, execSync } from "child_process";+import fs from "fs/promises";+import net from "net";+import path from "path";+import { fileURLToPath } from "url";++import { createTRPCClient, httpBatchLink } from "@trpc/client";+import superjson from "superjson";++import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";+import type {+ ZBookmark,+ ZBookmarkedLink,+} from "@karakeep/shared/types/bookmarks";+import type { ZBookmarkList } from "@karakeep/shared/types/lists";+import type { ZTagBasic } from "@karakeep/shared/types/tags";+import type { AppRouter } from "@karakeep/trpc/routers/_app";++type TrpcClient = ReturnType<typeof getTrpcClient>;+type CrawlStatus = NonNullable<ZBookmarkedLink["crawlStatus"]>;++const PASSWORD = "test1234";+const USER_DATASETS = {+ test1: {+ tagNames: ["research", "docs", "product"],+ urls: [+ "https://www.nasa.gov/",+ "https://www.nationalgeographic.com/",+ "https://www.theverge.com/",+ "https://www.wired.com/",+ "https://www.apple.com/ipad-pro/",+ "https://www.figma.com/",+ "https://linear.app/",+ "https://www.notion.com/",+ "https://stripe.com/",+ "https://www.shopify.com/",+ "https://developer.mozilla.org/en-US/docs/Web",+ "https://developer.mozilla.org/en-US/docs/Web/JavaScript",+ "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API",+ "https://developer.mozilla.org/en-US/docs/Web/CSS",+ "https://web.dev/learn/html",+ "https://web.dev/learn/css",+ "https://web.dev/learn/javascript",+ "https://www.raspberrypi.com/products/raspberry-pi-5/",+ "https://www.postgresql.org/docs/current/index.html",+ "https://docs.docker.com/get-started/",+ ],+ lists: {+ root: "Reading Queue",+ child: "Reference",+ smartTag: "Research Smart",+ smartFav: "Favorites Smart",+ },+ },+ test2: {+ tagNames: ["engineering", "design", "ops"],+ urls: [+ "https://github.com/features/actions",+ "https://www.cloudflare.com/",+ "https://web.dev/",+ "https://www.raspberrypi.com/products/raspberry-pi-5/",+ ],+ lists: {+ root: "Workbench",+ child: "Deep Dives",+ smartTag: "Engineering Smart",+ smartFav: "Archived Smart",+ },+ },+} as const;++type UserDataset = (typeof USER_DATASETS)[keyof typeof USER_DATASETS];++const USERS = [+ {+ email: "[email protected]",+ name: "Seed Test 1",+ datasetKey: "test1",+ },+ {+ email: "[email protected]",+ name: "Seed Test 2",+ datasetKey: "test2",+ },+ { email: "[email protected]", name: "Seed Test 3", datasetKey: null },+] as const;++interface SeededBookmark {+ id: string;+ url: string;+ title: string;+ crawlStatus: CrawlStatus | null;+}++interface SeededUserManifest {+ email: string;+ password: string;+ seeded: boolean;+ apiKeyName: string;+ counts: {+ bookmarks: number;+ tags: number;+ manualLists: number;+ smartLists: number;+ };+ bookmarks: SeededBookmark[];+ tags: string[];+ lists: Array<{+ name: string;+ type: "manual" | "smart";+ parentName: string | null;+ query: string | null;+ }>;+}++interface SnapshotManifest {+ createdAt: string;+ sourceCommit: string | null;+ archive: string;+ users: SeededUserManifest[];+}++function logStep(title: string): void {+ console.log(`\n== ${title}`);+}++function logInfo(message: string): void {+ console.log(` -- ${message}`);+}++function logSuccess(message: string): void {+ console.log(` OK ${message}`);+}++function logWarn(message: string): void {+ console.log(` !! ${message}`);+}++function sleep(ms: number): Promise<void> {+ return new Promise((resolve) => setTimeout(resolve, ms));+}++async function waitUntil(+ fn: () => Promise<boolean>,+ description: string,+ timeoutMs = 60000,+ intervalMs = 1000,+): Promise<void> {+ const start = Date.now();+ while (Date.now() - start < timeoutMs) {+ try {+ if (await fn()) return;+ } catch {+ // Retry until timeout. Services may still be warming up.+ }+ await sleep(intervalMs);+ }+ throw new Error(`${description} timed out after ${timeoutMs}ms`);+}++async function getRandomPort(): Promise<number> {+ const server = net.createServer();+ return new Promise<number>((resolve, reject) => {+ server.unref();+ server.on("error", reject);+ server.listen(0, () => {+ const address = server.address() as net.AddressInfo;+ server.close(() => resolve(address.port));+ });+ });+}++function getTrpcClient(+ apiKey?: string,+): ReturnType<typeof createTRPCClient<AppRouter>> {+ const port = process.env.KARAKEEP_PORT;+ if (!port) {+ throw new Error("KARAKEEP_PORT is not set. Did you start the containers?");+ }++ return createTRPCClient<AppRouter>({+ links: [+ httpBatchLink({+ transformer: superjson,+ url: `http://localhost:${port}/api/trpc`,+ headers() {+ return {+ authorization: apiKey ? `Bearer ${apiKey}` : undefined,+ };+ },+ }),+ ],+ });+}++async function waitForHealthy(port: number): Promise<void> {+ await waitUntil(+ async () => {+ const res = await fetch(`http://localhost:${port}/api/health`);+ return res.status === 200;+ },+ "Karakeep stack to become healthy",+ Number(process.env.SEED_SNAPSHOT_HEALTH_TIMEOUT_MS ?? 120_000),+ 1_000,+ );+}++async function captureDockerLogs(+ composeDir: string,+ composeEnv: NodeJS.ProcessEnv,+): Promise<void> {+ const logsDir = path.join(composeDir, "docker-logs");+ await fs.mkdir(logsDir, { recursive: true });++ for (const service of ["web", "meilisearch", "chrome"]) {+ try {+ execSync(+ `/bin/sh -c 'docker compose logs ${service} > "${logsDir}/${service}.log" 2>&1'`,+ {+ cwd: composeDir,+ env: composeEnv,+ stdio: "ignore",+ },+ );+ logInfo(`Captured logs for ${service}`);+ } catch (error) {+ logWarn(`Failed to capture logs for ${service}: ${error}`);+ }+ }+}++async function startContainers(composeDir: string, dataDir: string) {+ const port = await getRandomPort();+ const skipBuild =+ process.env.SEED_SNAPSHOT_NO_BUILD === "1" ||+ process.env.SEED_SNAPSHOT_SKIP_BUILD === "1";+ const buildArg = skipBuild ? "" : "--build";+ const composeEnv = {+ ...process.env,+ KARAKEEP_PORT: String(port),+ SEED_SNAPSHOT_DATA_DIR: dataDir,+ };++ logStep(`Starting docker compose on port ${port}`);+ execSync(`docker compose up ${buildArg} -d`, {+ cwd: composeDir,+ stdio: "inherit",+ env: composeEnv,+ });++ process.env.KARAKEEP_PORT = String(port);++ try {+ logInfo("Waiting for Karakeep to report healthy...");+ await waitForHealthy(port);+ await sleep(5_000);+ logSuccess("Containers are ready");+ } catch (error) {+ logWarn("Karakeep did not become healthy; stopping docker compose");+ await captureDockerLogs(composeDir, composeEnv);+ execSync("docker compose down", {+ cwd: composeDir,+ env: composeEnv,+ stdio: "inherit",+ });+ throw error;+ }++ let stopped = false;+ return {+ port,+ async stop(): Promise<void> {+ if (stopped) return;+ stopped = true;+ logStep("Collecting docker logs");+ await captureDockerLogs(composeDir, composeEnv);+ logStep("Stopping docker compose");+ execSync("docker compose down", {+ cwd: composeDir,+ env: composeEnv,+ stdio: "inherit",+ });+ },+ };+}++async function createUser(+ authlessClient: TrpcClient,+ email: string,+ name: string,+): Promise<TrpcClient> {+ logInfo(`Creating ${email}`);+ await authlessClient.users.create.mutate({+ name,+ email,+ password: PASSWORD,+ confirmPassword: PASSWORD,+ });++ const keyName = `seed-snapshot-${email}`;+ const { key } = await authlessClient.apiKeys.exchange.mutate({+ email,+ password: PASSWORD,+ keyName,+ });++ return getTrpcClient(key);+}++async function seedTags(+ client: TrpcClient,+ tagNames: readonly string[],+): Promise<ZTagBasic[]> {+ const tags: ZTagBasic[] = [];+ for (const name of tagNames) {+ tags.push(await client.tags.create.mutate({ name }));+ }+ return tags;+}++async function createLists(+ client: TrpcClient,+ dataset: UserDataset,+): Promise<{+ manualLists: ZBookmarkList[];+ smartLists: ZBookmarkList[];+}> {+ const root = await client.lists.create.mutate({+ name: dataset.lists.root,+ description: "Seed snapshot root manual list",+ icon: "book",+ type: "manual",+ });+ const child = await client.lists.create.mutate({+ name: dataset.lists.child,+ description: "Seed snapshot nested manual list",+ icon: "folder",+ type: "manual",+ parentId: root.id,+ });+ const smartTag = await client.lists.create.mutate({+ name: dataset.lists.smartTag,+ description: "Seed snapshot smart list by tag",+ icon: "tag",+ type: "smart",+ query: `tag:${dataset.tagNames[0]}`,+ });+ const smartFav = await client.lists.create.mutate({+ name: dataset.lists.smartFav,+ description: "Seed snapshot smart list by state",+ icon: "star",+ type: "smart",+ query: dataset.lists.smartFav.includes("Archived")+ ? "is:archived"+ : "is:fav",+ parentId: root.id,+ });++ return {+ manualLists: [root, child],+ smartLists: [smartTag, smartFav],+ };+}++async function createBookmarks(+ client: TrpcClient,+ dataset: UserDataset,+ tags: ZTagBasic[],+ lists: ZBookmarkList[],+): Promise<SeededBookmark[]> {+ const bookmarks: SeededBookmark[] = [];+ for (const [index, url] of dataset.urls.entries()) {+ const bookmark = await client.bookmarks.createBookmark.mutate({+ type: BookmarkTypes.LINK,+ url,+ title: `Seed snapshot bookmark ${index + 1}`,+ source: "api",+ summary: `Seeded bookmark for ${url}`,+ favourited: index % 2 === 0,+ archived: index === dataset.urls.length - 1,+ createdAt: new Date(Date.UTC(2025, 0, index + 1, 12, 0, 0)),+ });++ const attachedTags = [+ tags[index % tags.length],+ tags[(index + 1) % tags.length],+ ];+ await client.bookmarks.updateTags.mutate({+ bookmarkId: bookmark.id,+ attach: attachedTags.map((tag) => ({+ tagId: tag.id,+ tagName: tag.name,+ })),+ detach: [],+ });++ await client.lists.addToList.mutate({+ listId: lists[index % lists.length].id,+ bookmarkId: bookmark.id,+ });++ bookmarks.push({+ id: bookmark.id,+ url,+ title: bookmark.title ?? `Seed snapshot bookmark ${index + 1}`,+ crawlStatus: null,+ });+ }+ return bookmarks;+}++function getTerminalCrawlStatus(bookmark: ZBookmark): CrawlStatus | null {+ if (bookmark.content.type !== BookmarkTypes.LINK) {+ return null;+ }+ const status = bookmark.content.crawlStatus ?? null;+ return status === "success" || status === "failure" ? status : null;+}++async function waitForCrawls(+ client: TrpcClient,+ bookmarks: SeededBookmark[],+): Promise<void> {+ const pending = new Set(bookmarks.map((bookmark) => bookmark.id));+ await waitUntil(+ async () => {+ for (const bookmark of bookmarks) {+ if (!pending.has(bookmark.id)) continue;++ const latest = await client.bookmarks.getBookmark.query({+ bookmarkId: bookmark.id,+ });+ const terminalStatus = getTerminalCrawlStatus(latest);+ if (terminalStatus) {+ bookmark.crawlStatus = terminalStatus;+ pending.delete(bookmark.id);+ logInfo(`${bookmark.url} crawled with status ${terminalStatus}`);+ }+ }+ return pending.size === 0;+ },+ "Seed bookmark crawls to reach terminal status",+ Number(process.env.SEED_SNAPSHOT_CRAWL_TIMEOUT_MS ?? 180_000),+ 3_000,+ );+}++async function seedUser(+ client: TrpcClient,+ user: (typeof USERS)[number],+ dataset: UserDataset,+): Promise<SeededUserManifest> {+ logStep(`Seeding ${user.email}`);+ const tags = await seedTags(client, dataset.tagNames);+ const { manualLists, smartLists } = await createLists(client, dataset);+ const bookmarks = await createBookmarks(client, dataset, tags, manualLists);+ await waitForCrawls(client, bookmarks);++ return {+ email: user.email,+ password: PASSWORD,+ seeded: true,+ apiKeyName: `seed-snapshot-${user.email}`,+ counts: {+ bookmarks: bookmarks.length,+ tags: tags.length,+ manualLists: manualLists.length,+ smartLists: smartLists.length,+ },+ bookmarks,+ tags: tags.map((tag) => tag.name),+ lists: [...manualLists, ...smartLists].map((list) => ({+ name: list.name,+ type: list.type,+ parentName:+ list.parentId === manualLists[0]?.id ? manualLists[0].name : null,+ query: list.query ?? null,+ })),+ };+}++async function createUsersAndSeedData(): Promise<SeededUserManifest[]> {+ const authlessClient = getTrpcClient();+ const manifests: SeededUserManifest[] = [];++ logStep("Creating seed users");+ for (const user of USERS) {+ const client = await createUser(authlessClient, user.email, user.name);+ if (user.datasetKey) {+ manifests.push(+ await seedUser(client, user, USER_DATASETS[user.datasetKey]),+ );+ } else {+ manifests.push({+ email: user.email,+ password: PASSWORD,+ seeded: false,+ apiKeyName: `seed-snapshot-${user.email}`,+ counts: {+ bookmarks: 0,+ tags: 0,+ manualLists: 0,+ smartLists: 0,+ },+ bookmarks: [],+ tags: [],+ lists: [],+ });+ logSuccess(`${user.email} left empty`);+ }+ }++ return manifests;+}++function formatSnapshotTimestamp(date: Date): string {+ const pad = (num: number) => String(num).padStart(2, "0");+ return [+ date.getUTCFullYear(),+ "-",+ pad(date.getUTCMonth() + 1),+ "-",+ pad(date.getUTCDate()),+ "-",+ pad(date.getUTCHours()),+ pad(date.getUTCMinutes()),+ pad(date.getUTCSeconds()),+ ].join("");+}++function getSourceCommit(repoRoot: string): string | null {+ try {+ return execSync("git rev-parse HEAD", {+ cwd: repoRoot,+ encoding: "utf8",+ stdio: ["ignore", "pipe", "ignore"],+ }).trim();+ } catch {+ return null;+ }+}++async function writeSnapshot(params: {+ dataDir: string;+ repoRoot: string;+ users: SeededUserManifest[];+}): Promise<void> {+ const snapshotsDir = path.join(params.repoRoot, "snapshots");+ await fs.mkdir(snapshotsDir, { recursive: true });++ const createdAt = new Date();+ const timestamp = formatSnapshotTimestamp(createdAt);+ const archiveName = `seed-data-${timestamp}.tar.gz`;+ const manifestName = `seed-data-${timestamp}.json`;+ const archivePath = path.join(snapshotsDir, archiveName);+ const manifestPath = path.join(snapshotsDir, manifestName);++ logStep("Archiving data directory");+ execFileSync(+ "tar",+ ["--exclude", "._*", "-czf", archivePath, "-C", params.dataDir, "."],+ {+ cwd: params.repoRoot,+ env: {+ ...process.env,+ COPYFILE_DISABLE: "1",+ },+ stdio: "inherit",+ },+ );++ const manifest: SnapshotManifest = {+ createdAt: createdAt.toISOString(),+ sourceCommit: getSourceCommit(params.repoRoot),+ archive: archiveName,+ users: params.users,+ };+ await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);++ logSuccess(`Wrote ${archivePath}`);+ logSuccess(`Wrote ${manifestPath}`);+}++async function main(): Promise<void> {+ const __dirname = path.dirname(fileURLToPath(import.meta.url));+ const packageRoot = path.join(__dirname, "..");+ const repoRoot = path.join(packageRoot, "..", "..");+ const composeDir = packageRoot;+ const dataDir = path.join(packageRoot, ".tmp", "data");++ await fs.rm(dataDir, { force: true, recursive: true });+ await fs.mkdir(dataDir, { recursive: true });+ await fs.chmod(dataDir, 0o777);++ const running = await startContainers(composeDir, dataDir);++ const stopContainers = async () => {+ await running.stop();+ };++ const handleSignal = async (signal: NodeJS.Signals) => {+ logWarn(`Received ${signal}, shutting down...`);+ await stopContainers();+ process.exit(1);+ };++ process.on("SIGINT", handleSignal);+ process.on("SIGTERM", handleSignal);++ try {+ const users = await createUsersAndSeedData();+ await stopContainers();+ await writeSnapshot({ dataDir, repoRoot, users });+ logSuccess("Seed snapshot complete");+ } catch (error) {+ logWarn("Seed snapshot failed");+ console.error(error);+ await stopContainers();+ process.exitCode = 1;+ }+}++main();diff --git a/tools/seed-snapshot/tsconfig.json b/tools/seed-snapshot/tsconfig.jsonnew file mode 100644index 00000000..ae9547cb--- /dev/null+++ b/tools/seed-snapshot/tsconfig.json@@ -0,0 +1,9 @@+{+ "$schema": "https://json.schemastore.org/tsconfig",+ "extends": "@karakeep/tsconfig/node.json",+ "include": ["src/**/*.ts"],+ "exclude": ["node_modules"],+ "compilerOptions": {+ "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"+ }+}