A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
feat: various improvements to the CLI: pretty print results, list creation, highlights, etc
Mohamed Bassemd334df92df1b
diff --git a/apps/cli/src/commands/bookmarks.ts b/apps/cli/src/commands/bookmarks.tsindex e2e8efb6..fc220ead 100644--- a/apps/cli/src/commands/bookmarks.ts+++ b/apps/cli/src/commands/bookmarks.ts@@ -1,5 +1,7 @@import * as fs from "node:fs";+import * as path from "node:path";import { addToList } from "@/commands/lists";+import { getGlobalOptions } from "@/lib/globals";import {printError,printObject,@@ -8,6 +10,7 @@ import {} from "@/lib/output";import { getAPIClient } from "@/lib/trpc";import { Command } from "@commander-js/extra-typings";+import chalk from "chalk";import type { ZBookmark } from "@karakeep/shared/types/bookmarks";import {@@ -35,8 +38,108 @@ function normalizeBookmark(bookmark: ZBookmark): Bookmark {};}-function printBookmark(bookmark: ZBookmark) {- printObject(normalizeBookmark(bookmark));+function printBookmarkDetail(b: ZBookmark) {+ if (getGlobalOptions().json) {+ printObject(normalizeBookmark(b));+ return;+ }++ const title = getBookmarkTitle(b);+ const url = getBookmarkUrl(b);+ const tags = b.tags.map((t) => t.name).join(", ");+ const type =+ b.content.type === BookmarkTypes.ASSET+ ? `asset (${b.content.assetType})`+ : b.content.type;++ console.log(chalk.bold(title));+ console.log(chalk.dim(` Id: ${b.id}`));+ console.log(chalk.dim(` Type: ${type}`));+ if (url) console.log(` URL: ${chalk.cyan(url)}`);+ if (tags) console.log(` Tags: ${tags}`);+ if (b.archived) console.log(` Archived: yes`);+ if (b.favourited) console.log(` Favourited: yes`);+ console.log(` Created: ${b.createdAt.toISOString()}`);+ if (b.modifiedAt) console.log(` Modified: ${b.modifiedAt.toISOString()}`);+ if (b.source) console.log(` Source: ${b.source}`);+ if (b.note) console.log(` Note: ${b.note}`);+ if (b.summary) console.log(` Summary: ${b.summary}`);++ if (b.content.type === BookmarkTypes.LINK) {+ if (b.content.author) console.log(` Author: ${b.content.author}`);+ if (b.content.publisher)+ console.log(` Publisher: ${b.content.publisher}`);+ if (b.content.description)+ console.log(` Description: ${b.content.description}`);+ if (b.content.crawlStatus)+ console.log(` Crawl: ${b.content.crawlStatus}`);+ }++ if (b.content.type === BookmarkTypes.TEXT) {+ console.log();+ console.log(b.content.text);+ }++ if (b.assets.length > 0) {+ const serverAddr = getGlobalOptions().serverAddr;+ console.log(` Attachments:`);+ for (const asset of b.assets) {+ const name = asset.fileName ?? asset.assetType;+ const assetUrl = `${serverAddr}/api/assets/${asset.id}`;+ console.log(` - ${name} ${chalk.cyan(assetUrl)}`);+ }+ }+ console.log();+}++function getBookmarkTitle(bookmark: ZBookmark): string {+ if (bookmark.title) return bookmark.title;+ switch (bookmark.content.type) {+ case BookmarkTypes.LINK:+ return bookmark.content.title ?? bookmark.content.url;+ case BookmarkTypes.TEXT:+ return bookmark.content.text.replaceAll(/\s+/g, " ").substring(0, 50);+ case BookmarkTypes.ASSET:+ return bookmark.content.fileName ?? "asset";+ default:+ return "";+ }+}++function getBookmarkUrl(bookmark: ZBookmark): string {+ if (bookmark.content.type === BookmarkTypes.LINK) {+ return bookmark.content.url;+ }+ if (bookmark.content.type === BookmarkTypes.ASSET) {+ const serverAddr = getGlobalOptions().serverAddr;+ return `${serverAddr}/api/assets/${bookmark.content.assetId}`;+ }+ return "";+}++function printBookmarkCard(b: ZBookmark) {+ const title = getBookmarkTitle(b);+ const url = getBookmarkUrl(b);+ const tags = b.tags.map((t) => t.name).join(", ");++ console.log(chalk.bold(title));+ console.log(chalk.dim(` Id: ${b.id}`));+ const type =+ b.content.type === BookmarkTypes.ASSET+ ? `asset (${b.content.assetType})`+ : b.content.type;+ console.log(chalk.dim(` Type: ${type}`));+ console.log(chalk.dim(` Created: ${b.createdAt.toISOString()}`));+ if (url) console.log(` URL: ${chalk.cyan(url)}`);+ if (b.content.type === BookmarkTypes.LINK) {+ if (b.content.author) console.log(` Author: ${b.content.author}`);+ if (b.content.publisher) console.log(` Publisher: ${b.content.publisher}`);+ }+ if (tags) console.log(` Tags: ${tags}`);+ if (b.archived) console.log(` Archived: yes`);+ if (b.favourited) console.log(` Favourited: yes`);+ if (b.note) console.log(` Note: ${b.note}`);+ console.log();}bookmarkCmd@@ -54,6 +157,12 @@ bookmarkCmdcollect<string>,[],)+ .option(+ "--asset <file>",+ "the file path of an asset (image or pdf) to add. Specify multiple times to add multiple assets",+ collect<string>,+ [],+ ).option("--stdin", "reads the data from stdin and store it as a note").option("--list-id <id>",@@ -128,6 +237,51 @@ bookmarkCmd);}+ const globals = getGlobalOptions();+ for (const filePath of opts.asset) {+ promises.push(+ (async () => {+ const fileBuffer = fs.readFileSync(filePath);+ const fileName = path.basename(filePath);+ const formData = new FormData();+ formData.append("file", new Blob([fileBuffer]), fileName);++ const uploadResp = await fetch(+ `${globals.serverAddr}/api/v1/assets`,+ {+ method: "POST",+ headers: {+ authorization: `Bearer ${globals.apiKey}`,+ },+ body: formData,+ },+ );+ if (!uploadResp.ok) {+ throw new Error(+ `Upload failed: ${uploadResp.status} ${uploadResp.statusText}`,+ );+ }+ const { assetId, contentType } = (await uploadResp.json()) as {+ assetId: string;+ contentType: string;+ };+ const assetType = contentType === "application/pdf" ? "pdf" : "image";++ const bookmark = await api.bookmarks.createBookmark.mutate({+ type: BookmarkTypes.ASSET,+ assetType,+ assetId,+ fileName,+ title: opts.title,+ source: "cli",+ });+ results.push(normalizeBookmark(bookmark));+ })().catch(+ printError(`Failed to add an asset bookmark for file "${filePath}"`),+ ),+ );+ }+await Promise.allSettled(promises);printObject(results);@@ -152,7 +306,7 @@ bookmarkCmdconst api = getAPIClient();await api.bookmarks.getBookmark.query({ bookmarkId: id, includeContent: opts.includeContent })- .then(printBookmark)+ .then(printBookmarkDetail).catch(printError(`Failed to get the bookmark with id "${id}"`));});@@ -244,41 +398,79 @@ bookmarkCmdbookmarkCmd.command("list")- .description("list all bookmarks")+ .description("list bookmarks").option("--include-archived","If set, archived bookmarks will be fetched as well",false,).option("--list-id <id>", "if set, only items from that list will be fetched")+ .option("--tag-id <id>", "if set, only items with that tag will be fetched")+ .option(+ "--feed-id <id>",+ "if set, only items from that RSS feed will be fetched",+ ).option("--include-content","include full bookmark content in results",false,)+ .option(+ "--limit <limit>",+ `number of bookmarks per page (max ${MAX_NUM_BOOKMARKS_PER_PAGE})`,+ (v: string) => Math.min(parseInt(v, 10), MAX_NUM_BOOKMARKS_PER_PAGE),+ 20,+ )+ .option("--all", "fetch all bookmarks (paginate through all pages)", false)+ .option("--cursor <cursor>", "cursor from a previous request for pagination").action(async (opts) => {const api = getAPIClient();const request = {archived: opts.includeArchived ? undefined : false,listId: opts.listId,- limit: MAX_NUM_BOOKMARKS_PER_PAGE,+ tagId: opts.tagId,+ rssFeedId: opts.feedId,+ limit: opts.limit,useCursorV2: true,includeContent: opts.includeContent,+ cursor: opts.cursor+ ? JSON.parse(Buffer.from(opts.cursor, "base64").toString(), (k, v) =>+ k === "createdAt" ? new Date(v) : v,+ )+ : undefined,};try {let resp = await api.bookmarks.getBookmarks.query(request);let results: ZBookmark[] = resp.bookmarks;- while (resp.nextCursor) {- resp = await api.bookmarks.getBookmarks.query({- ...request,- cursor: resp.nextCursor,- });- results = [...results, ...resp.bookmarks];+ if (opts.all) {+ while (resp.nextCursor) {+ resp = await api.bookmarks.getBookmarks.query({+ ...request,+ cursor: resp.nextCursor,+ });+ results = [...results, ...resp.bookmarks];+ }+ }++ const nextCursor =+ !opts.all && resp.nextCursor+ ? Buffer.from(JSON.stringify(resp.nextCursor)).toString("base64")+ : undefined;++ if (getGlobalOptions().json) {+ printObject(+ { bookmarks: results.map(normalizeBookmark), nextCursor },+ { maxArrayLength: null },+ );+ } else {+ results.forEach(printBookmarkCard);+ if (nextCursor) {+ console.log(`Next cursor: ${chalk.dim(nextCursor)}`);+ }}- printObject(results.map(normalizeBookmark), { maxArrayLength: null });} catch {printStatusMessage(false, "Failed to query bookmarks");}@@ -314,6 +506,7 @@ bookmarkCmdfalse,).option("--all", "fetch all results (paginate through all pages)", false)+ .option("--cursor <cursor>", "cursor from a previous request for pagination").action(async (query, opts) => {const api = getAPIClient();@@ -322,13 +515,17 @@ bookmarkCmdlimit: opts.limit,sortOrder: opts.sortOrder as "relevance" | "asc" | "desc",includeContent: opts.includeContent,+ cursor: opts.cursor+ ? JSON.parse(Buffer.from(opts.cursor, "base64").toString(), (k, v) =>+ k === "createdAt" ? new Date(v) : v,+ )+ : undefined,};try {let resp = await api.bookmarks.searchBookmarks.query(request);let results: ZBookmark[] = resp.bookmarks;- // If --all flag is set, fetch all pagesif (opts.all) {while (resp.nextCursor) {resp = await api.bookmarks.searchBookmarks.query({@@ -339,7 +536,22 @@ bookmarkCmd}}- printObject(results.map(normalizeBookmark), { maxArrayLength: null });+ const nextCursor =+ !opts.all && resp.nextCursor+ ? Buffer.from(JSON.stringify(resp.nextCursor)).toString("base64")+ : undefined;++ if (getGlobalOptions().json) {+ printObject(+ { bookmarks: results.map(normalizeBookmark), nextCursor },+ { maxArrayLength: null },+ );+ } else {+ results.forEach(printBookmarkCard);+ if (nextCursor) {+ console.log(`Next cursor: ${chalk.dim(nextCursor)}`);+ }+ }} catch (error) {printStatusMessage(false, "Failed to search bookmarks");if (error instanceof Error) {diff --git a/apps/cli/src/commands/highlights.ts b/apps/cli/src/commands/highlights.tsnew file mode 100644index 00000000..b3d7cd26--- /dev/null+++ b/apps/cli/src/commands/highlights.ts@@ -0,0 +1,133 @@+import { getGlobalOptions } from "@/lib/globals";+import {+ printError,+ printErrorMessageWithReason,+ printObject,+} from "@/lib/output";+import { getAPIClient } from "@/lib/trpc";+import { Command } from "@commander-js/extra-typings";+import chalk from "chalk";++export const highlightsCmd = new Command()+ .name("highlights")+ .description("manipulating highlights");++function printHighlightCard(h: {+ id: string;+ bookmarkId: string;+ text: string | null;+ note: string | null;+ color: string;+ createdAt: Date;+}) {+ console.log(chalk.bold(h.text ?? "(no text)"));+ console.log(chalk.dim(` Id: ${h.id}`));+ console.log(chalk.dim(` Bookmark: ${h.bookmarkId}`));+ console.log(chalk.dim(` Color: ${h.color}`));+ console.log(chalk.dim(` Created: ${h.createdAt.toISOString()}`));+ if (h.note) console.log(` Note: ${h.note}`);+ console.log();+}++highlightsCmd+ .command("list")+ .description("list all highlights")+ .option("--bookmark <id>", "list highlights for a specific bookmark")+ .option(+ "--limit <limit>",+ "number of highlights per page",+ (v: string) => parseInt(v, 10),+ 20,+ )+ .option("--all", "fetch all highlights (paginate through all pages)", false)+ .option("--cursor <cursor>", "cursor from a previous request for pagination")+ .action(async (opts) => {+ const api = getAPIClient();++ try {+ if (opts.bookmark) {+ const resp = await api.highlights.getForBookmark.query({+ bookmarkId: opts.bookmark,+ });+ if (getGlobalOptions().json) {+ printObject(resp.highlights);+ } else {+ resp.highlights.forEach(printHighlightCard);+ }+ } else {+ const request = {+ limit: opts.limit,+ cursor: opts.cursor+ ? JSON.parse(+ Buffer.from(opts.cursor, "base64").toString(),+ (k, v) => (k === "createdAt" ? new Date(v) : v),+ )+ : undefined,+ };++ let resp = await api.highlights.getAll.query(request);+ let results = resp.highlights;++ if (opts.all) {+ while (resp.nextCursor) {+ resp = await api.highlights.getAll.query({+ ...request,+ cursor: resp.nextCursor,+ });+ results = [...results, ...resp.highlights];+ }+ }++ const nextCursor =+ !opts.all && resp.nextCursor+ ? Buffer.from(JSON.stringify(resp.nextCursor)).toString("base64")+ : undefined;++ if (getGlobalOptions().json) {+ printObject({ highlights: results, nextCursor });+ } else {+ results.forEach(printHighlightCard);+ if (nextCursor) {+ console.log(`Next cursor: ${chalk.dim(nextCursor)}`);+ }+ }+ }+ } catch (error) {+ printErrorMessageWithReason("Failed to list highlights", error as object);+ }+ });++highlightsCmd+ .command("get")+ .description("get a highlight by id")+ .argument("<id>", "the id of the highlight")+ .action(async (id) => {+ const api = getAPIClient();++ try {+ const highlight = await api.highlights.get.query({ highlightId: id });+ if (getGlobalOptions().json) {+ printObject(highlight);+ } else {+ printHighlightCard(highlight);+ }+ } catch (error) {+ printErrorMessageWithReason(+ `Failed to get highlight with id "${id}"`,+ error as object,+ );+ }+ });++highlightsCmd+ .command("delete")+ .description("delete a highlight")+ .argument("<id>", "the id of the highlight")+ .action(async (id) => {+ const api = getAPIClient();++ await api.highlights.delete+ .mutate({ highlightId: id })+ .then(() => console.log(`Successfully deleted highlight "${id}"`))+ .catch(printError(`Failed to delete highlight with id "${id}"`));+ });diff --git a/apps/cli/src/commands/lists.ts b/apps/cli/src/commands/lists.tsindex 1d9341d7..0b9c38fa 100644--- a/apps/cli/src/commands/lists.ts+++ b/apps/cli/src/commands/lists.ts@@ -7,6 +7,7 @@ import {} from "@/lib/output";import { getAPIClient } from "@/lib/trpc";import { Command } from "@commander-js/extra-typings";+import chalk from "chalk";import { getBorderCharacters, table } from "table";import { listsToTree } from "@karakeep/shared/utils/listUtils";@@ -22,18 +23,22 @@ listsCmdconst api = getAPIClient();try {- const resp = await api.lists.list.query();+ const [resp, statsResp] = await Promise.all([+ api.lists.list.query(),+ api.lists.stats.query(),+ ]);if (getGlobalOptions().json) {printObject(resp);} else {const { allPaths } = listsToTree(resp.lists);- const data: string[][] = [["Id", "Name"]];+ const data: string[][] = [["Id", "Name", "Description", "Bookmarks"]];allPaths.forEach((path) => {const name = path.map((p) => `${p.icon} ${p.name}`).join(" / ");- const id = path[path.length - 1].id;- data.push([id, name]);+ const leaf = path[path.length - 1];+ const count = statsResp.stats.get(leaf.id) ?? 0;+ data.push([leaf.id, name, leaf.description ?? "", count.toString()]);});console.log(table(data, {@@ -47,6 +52,34 @@ listsCmd}});+listsCmd+ .command("create")+ .description("creates a new list")+ .requiredOption("--name <name>", "the name of the list")+ .requiredOption("--icon <icon>", "the icon of the list (one emoji)")+ .option("--type <type>", "the type of the list (manual or smart)", "manual")+ .option("--description <description>", "the description of the list")+ .option(+ "--query <query>",+ "the search query for smart lists (required for smart lists)",+ )+ .option("--parent-id <id>", "the id of the parent list")+ .action(async (opts) => {+ const api = getAPIClient();++ await api.lists.create+ .mutate({+ name: opts.name,+ icon: opts.icon,+ type: opts.type as "manual" | "smart",+ description: opts.description,+ query: opts.query,+ parentId: opts.parentId,+ })+ .then(printObject)+ .catch(printError("Failed to create list"));+ });+listsCmd.command("delete").description("deletes a list")@@ -84,36 +117,28 @@ export async function addToList(listId: string, bookmarkId: string) {listsCmd.command("get")- .description("gets all the ids of the bookmarks assigned to the list")- .requiredOption("--list <id>", "the id of the list")- .option(- "--include-content",- "include full bookmark content in results",- false,- )- .action(async (opts) => {+ .description("get a list by id")+ .argument("<id>", "the id of the list")+ .action(async (id) => {const api = getAPIClient();try {- let resp = await api.bookmarks.getBookmarks.query({- listId: opts.list,- includeContent: opts.includeContent,- });- let results: string[] = resp.bookmarks.map((b) => b.id);- while (resp.nextCursor) {- resp = await api.bookmarks.getBookmarks.query({- listId: opts.list,- cursor: resp.nextCursor,- includeContent: opts.includeContent,- });- results = [...results, ...resp.bookmarks.map((b) => b.id)];+ const list = await api.lists.get.query({ listId: id });+ if (getGlobalOptions().json) {+ printObject(list);+ } else {+ console.log(chalk.bold(`${list.icon} ${list.name}`));+ console.log(chalk.dim(` Id: ${list.id}`));+ console.log(chalk.dim(` Type: ${list.type}`));+ if (list.description) console.log(` Description: ${list.description}`);+ if (list.query) console.log(` Query: ${list.query}`);+ if (list.parentId) console.log(` Parent: ${list.parentId}`);+ console.log(` Public: ${list.public ? "yes" : "no"}`);+ console.log(` Role: ${list.userRole}`);+ if (list.hasCollaborators) console.log(` Collaborators: yes`);+ console.log();}-- printObject(results);} catch (error) {- printErrorMessageWithReason(- "Failed to get the ids of the bookmarks in the list",- error as object,- );+ printErrorMessageWithReason("Failed to get the list", error as object);}});diff --git a/apps/cli/src/commands/tags.ts b/apps/cli/src/commands/tags.tsindex b15fef90..e2f4deae 100644--- a/apps/cli/src/commands/tags.ts+++ b/apps/cli/src/commands/tags.ts@@ -7,6 +7,7 @@ import {} from "@/lib/output";import { getAPIClient } from "@/lib/trpc";import { Command } from "@commander-js/extra-typings";+import chalk from "chalk";import { getBorderCharacters, table } from "table";export const tagsCmd = new Command()@@ -42,6 +43,73 @@ tagsCmd}});+tagsCmd+ .command("get")+ .description("get a tag by id or name")+ .argument("[id]", "the id of the tag")+ .option("--name <name>", "the name of the tag")+ .action(async (id, opts) => {+ if (!id && !opts.name) {+ console.error("Error: either <id> or --name must be provided");+ process.exit(1);+ }++ const api = getAPIClient();++ try {+ let tagId = id;+ if (!tagId) {+ const tags = (await api.tags.list.query({ nameContains: opts.name }))+ .tags;+ const exactMatch = tags.find((t) => t.name === opts.name);+ if (!exactMatch) {+ console.error(`No tag found with the name "${opts.name}"`);+ process.exit(1);+ }+ tagId = exactMatch.id;+ }++ const tag = await api.tags.get.query({ tagId });+ if (getGlobalOptions().json) {+ printObject(tag);+ } else {+ console.log(chalk.bold(tag.name));+ console.log(chalk.dim(` Id: ${tag.id}`));+ console.log(` Bookmarks: ${tag.numBookmarks}`);+ const byType = tag.numBookmarksByAttachedType;+ if (byType.ai || byType.human) {+ const parts = [];+ if (byType.human) parts.push(`${byType.human} by human`);+ if (byType.ai) parts.push(`${byType.ai} by AI`);+ console.log(` Breakdown: ${parts.join(", ")}`);+ }+ console.log();+ }+ } catch (error) {+ printErrorMessageWithReason("Failed to get the tag", error as object);+ }+ });++tagsCmd+ .command("merge")+ .description("merge tags into a target tag")+ .requiredOption("--into <id>", "the id of the tag to merge into")+ .requiredOption(+ "--from <ids...>",+ "the ids of the tags to merge from (space separated)",+ )+ .action(async (opts) => {+ const api = getAPIClient();++ await api.tags.merge+ .mutate({+ intoTagId: opts.into,+ fromTagIds: opts.from,+ })+ .then(printObject)+ .catch(printError("Failed to merge tags"));+ });+tagsCmd.command("delete").description("delete a tag")diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 8158c0b8..e3b73e05 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -1,6 +1,7 @@import { adminCmd } from "@/commands/admin";import { bookmarkCmd } from "@/commands/bookmarks";import { dumpCmd } from "@/commands/dump";+import { highlightsCmd } from "@/commands/highlights";import { listsCmd } from "@/commands/lists";import { migrateCmd } from "@/commands/migrate";import { tagsCmd } from "@/commands/tags";@@ -34,6 +35,7 @@ const program = new Command()program.addCommand(adminCmd);program.addCommand(bookmarkCmd);+program.addCommand(highlightsCmd);program.addCommand(listsCmd);program.addCommand(tagsCmd);program.addCommand(whoamiCmd);