⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
auto tag existing links & use meta description
Isaac9d8d5f0fa00f
diff --git a/.env.sample b/.env.sampleindex 6615c791..5781f1e8 100644--- a/.env.sample+++ b/.env.sample@@ -41,6 +41,7 @@ NEXT_PUBLIC_RSS_POLLING_INTERVAL_MINUTES=RSS_SUBSCRIPTION_LIMIT_PER_USER=# AI Settings+AI_TAGGING_SCRIPT_INTERVAL=NEXT_PUBLIC_OLLAMA_ENDPOINT_URL=OLLAMA_MODEL=diff --git a/components/Preservation/Preservation.tsx b/components/Preservation/Preservation.tsxindex 83148815..0048c3d6 100644--- a/components/Preservation/Preservation.tsx+++ b/components/Preservation/Preservation.tsx@@ -114,7 +114,7 @@ export default function Preservation({className={clsx("bg-base-200",!isExpanded &&- `mx-auto relative p-2 h-full overflow-auto rounded-md sm:rounded-none`,+ `mx-auto relative p-2 h-full overflow-auto rounded-md sm:rounded-none`,isExpanded && "fixed inset-0 w-screen h-screen z-[9999]")}style={{@@ -129,7 +129,7 @@ export default function Preservation({className={clsx("py-1 px-2 cursor-pointer duration-100 rounded-lg font-semibold text-center flex justify-between items-center w-6",format === ArchivedFormat.readability &&- "bg-primary bg-opacity-50"+ "bg-primary bg-opacity-50")}onClick={() => setFormat(ArchivedFormat.readability)}>@@ -143,7 +143,7 @@ export default function Preservation({className={clsx("py-1 px-2 cursor-pointer duration-100 rounded-lg font-semibold text-center flex justify-between items-center w-6",format === ArchivedFormat.monolith &&- "bg-primary bg-opacity-50"+ "bg-primary bg-opacity-50")}onClick={() => setFormat(ArchivedFormat.monolith)}>@@ -157,9 +157,9 @@ export default function Preservation({className={clsx("py-1 px-2 cursor-pointer duration-100 rounded-lg font-semibold text-center flex justify-between items-center w-6",format ===- (link?.image?.endsWith("png")- ? ArchivedFormat.png- : ArchivedFormat.jpeg) && "bg-primary bg-opacity-50"+ (link?.image?.endsWith("png")+ ? ArchivedFormat.png+ : ArchivedFormat.jpeg) && "bg-primary bg-opacity-50")}onClick={() =>setFormat(@@ -279,29 +279,29 @@ const RenderFormat = ({const style = isExpanded? {- position: "fixed" as const,- top: 0,- left: 0,- width: "100vw",- height: "100vh",- zIndex: 9999,- transitionProperty:- delayPassed && !shouldReduceMotion- ? "width, height, top, left"- : "none",- }+ position: "fixed" as const,+ top: 0,+ left: 0,+ width: "100vw",+ height: "100vh",+ zIndex: 9999,+ transitionProperty:+ delayPassed && !shouldReduceMotion+ ? "width, height, top, left"+ : "none",+ }: {- zIndex: 40,- position: "absolute" as const,- top: (containerRect?.top ?? 0) + window.scrollY + topPadding,- left: (containerRect?.left ?? 0) + window.scrollX,- width: containerRect?.width ?? 0,- height: containerRect?.height ? containerRect.height - topPadding : 0,- transitionProperty:- delayPassed && !shouldReduceMotion- ? "width, height, top, left"- : "none",- };+ zIndex: 40,+ position: "absolute" as const,+ top: (containerRect?.top ?? 0) + window.scrollY + topPadding,+ left: (containerRect?.left ?? 0) + window.scrollX,+ width: containerRect?.width ?? 0,+ height: containerRect?.height ? containerRect.height - topPadding : 0,+ transitionProperty:+ delayPassed && !shouldReduceMotion+ ? "width, height, top, left"+ : "none",+ };return ReactDOM.createPortal(<divdiff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.tsindex bb3ed339..e25ad66f 100644--- a/lib/api/archiveHandler.ts+++ b/lib/api/archiveHandler.ts@@ -86,13 +86,12 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {await page.goto(link.url, { waitUntil: "domcontentloaded" });- const content = await page.content();-- // Preview- if (!link.preview) await handleArchivePreview(link, page);+ const metaDescription = await page.evaluate(() => {+ const description = document.querySelector('meta[name="description"]');+ return description?.getAttribute('content') ?? undefined;+ });- // Readability- if (!link.readable) await handleReadablility(content, link);+ const content = await page.content();// Auto-taggingif (@@ -104,7 +103,13 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {process.env.ANTHROPIC_API_KEY))- await autoTagLink(user, link.id);+ await autoTagLink(user, link.id, metaDescription);++ // Preview+ if (!link.preview) await handleArchivePreview(link, page);++ // Readability+ if (!link.readable) await handleReadablility(content, link);// Screenshot/PDFif (@@ -189,7 +194,7 @@ async function determineLinkType(}// Construct browser launch options based on environment variables.-function getBrowserOptions(): LaunchOptions {+export function getBrowserOptions(): LaunchOptions {let browserOptions: LaunchOptions = {};if (process.env.PROXY) {diff --git a/lib/api/autoTagLink.ts b/lib/api/autoTagLink.tsindex 40a5fdae..26aa056c 100644--- a/lib/api/autoTagLink.ts+++ b/lib/api/autoTagLink.ts@@ -26,7 +26,7 @@ const getAIModel = () => {throw new Error("No AI provider configured");}-export default async function autoTagLink(user: User, linkId: number) {+export default async function autoTagLink(user: User, linkId: number, metaDescription: string | undefined) {const link = await prisma.link.findUnique({where: { id: linkId },});@@ -37,15 +37,15 @@ export default async function autoTagLink(user: User, linkId: number) {let prompt;- const textContent = link.textContent?.slice(0, 500) + "...";+ const promptText = metaDescription || link.textContent?.slice(0, 500) + "...";- if (!link.textContent)+ if (!promptText)return console.log("No text content to auto tag for link: ", link.url);if (user.aiTaggingMethod === AiTaggingMethod.GENERATE) {- prompt = generateTagsPrompt(textContent);+ prompt = generateTagsPrompt(promptText);} else {- prompt = predefinedTagsPrompt(textContent, user.aiPredefinedTags);+ prompt = predefinedTagsPrompt(promptText, user.aiPredefinedTags);}if (diff --git a/lib/api/controllers/links/postLink.ts b/lib/api/controllers/links/postLink.tsindex 76e0eb85..5cc885b0 100644--- a/lib/api/controllers/links/postLink.ts+++ b/lib/api/controllers/links/postLink.ts@@ -16,9 +16,8 @@ export default async function postLink(if (!dataValidation.success) {return {- response: `Error: ${- dataValidation.error.issues[0].message- } [${dataValidation.error.issues[0].path.join(", ")}]`,+ response: `Error: ${dataValidation.error.issues[0].message+ } [${dataValidation.error.issues[0].path.join(", ")}]`,status: 400,};}@@ -134,9 +133,8 @@ export default async function postLink(where: { id: newLink.id },data: {image: link.image- ? `archives/${newLink.collectionId}/${newLink.id}.${- link.image === "png" ? "png" : "jpeg"- }`+ ? `archives/${newLink.collectionId}/${newLink.id}.${link.image === "png" ? "png" : "jpeg"+ }`: undefined,},});diff --git a/lib/api/controllers/users/userId/updateUserById.ts b/lib/api/controllers/users/userId/updateUserById.tsindex 7f1e0892..69393d25 100644--- a/lib/api/controllers/users/userId/updateUserById.ts+++ b/lib/api/controllers/users/userId/updateUserById.ts@@ -19,9 +19,8 @@ export default async function updateUserById(if (!dataValidation.success) {return {- response: `Error: ${- dataValidation.error.issues[0].message- } [${dataValidation.error.issues[0].path.join(", ")}]`,+ response: `Error: ${dataValidation.error.issues[0].message+ } [${dataValidation.error.issues[0].path.join(", ")}]`,status: 400,};}@@ -33,18 +32,18 @@ export default async function updateUserById(id: { not: userId },OR: emailEnabled? [- {- username: data.username.toLowerCase(),- },- {- email: data.email?.toLowerCase(),- },- ]+ {+ username: data.username.toLowerCase(),+ },+ {+ email: data.email?.toLowerCase(),+ },+ ]: [- {- username: data.username.toLowerCase(),- },- ],+ {+ username: data.username.toLowerCase(),+ },+ ],},});@@ -203,6 +202,7 @@ export default async function updateUserById(),aiTaggingMethod: data.aiTaggingMethod,aiPredefinedTags: data.aiPredefinedTags,+ aiTagExistingLinks: data.aiTagExistingLinks,locale: i18n.locales.includes(data.locale || "") ? data.locale : "en",archiveAsScreenshot: data.archiveAsScreenshot,archiveAsMonolith: data.archiveAsMonolith,diff --git a/lib/shared/fetchTitleAndHeaders.ts b/lib/shared/fetchTitleAndHeaders.tsindex a7779646..fcfee319 100644--- a/lib/shared/fetchTitleAndHeaders.ts+++ b/lib/shared/fetchTitleAndHeaders.ts@@ -42,7 +42,7 @@ export default async function fetchTitleAndHeaders(url: string) {// regular expression to find the <title> taglet match = text.match(/<title.*>([^<]*)<\/title>/);- const title = match[1] || "";+ const title = match?.[1] || "";const headers = (response as Response)?.headers || null;return { title, headers };diff --git a/lib/shared/schemaValidation.ts b/lib/shared/schemaValidation.tsindex d549c9f5..7f80699d 100644--- a/lib/shared/schemaValidation.ts+++ b/lib/shared/schemaValidation.ts@@ -41,12 +41,12 @@ export const PostUserSchema = () => {username: emailEnabled? z.string().optional(): z- .string()- .trim()- .toLowerCase()- .min(3)- .max(50)- .regex(/^[a-z0-9_-]{3,50}$/),+ .string()+ .trim()+ .toLowerCase()+ .min(3)+ .max(50)+ .regex(/^[a-z0-9_-]{3,50}$/),invite: z.boolean().optional(),});};@@ -79,6 +79,7 @@ export const UpdateUserSchema = () => {dashboardRecentLinks: z.boolean().optional(),aiTaggingMethod: z.nativeEnum(AiTaggingMethod).optional(),aiPredefinedTags: z.array(z.string().max(20).trim()).max(20).optional(),+ aiTagExistingLinks: z.boolean().optional(),locale: z.string().max(20).optional(),isPrivate: z.boolean().optional(),preventDuplicateLinks: z.boolean().optional(),diff --git a/pages/settings/preference.tsx b/pages/settings/preference.tsxindex 7fd4208b..25849358 100644--- a/pages/settings/preference.tsx+++ b/pages/settings/preference.tsx@@ -44,6 +44,7 @@ export default function Appearance() {account.aiTaggingMethod);const [aiPredefinedTags, setAiPredefinedTags] = useState<string[]>();+ const [aiTagExistingLinks, setAiTagExistingLinks] = useState<boolean>(account.aiTagExistingLinks);const { data: config } = useConfig();@@ -58,6 +59,7 @@ export default function Appearance() {preventDuplicateLinks,aiTaggingMethod,aiPredefinedTags,+ aiTagExistingLinks,dashboardRecentLinks,dashboardPinnedLinks,});@@ -71,6 +73,7 @@ export default function Appearance() {preventDuplicateLinks,aiTaggingMethod,aiPredefinedTags,+ aiTagExistingLinks,dashboardRecentLinks,dashboardPinnedLinks,]);@@ -89,6 +92,7 @@ export default function Appearance() {setPreventDuplicateLinks(account.preventDuplicateLinks);setAiTaggingMethod(account.aiTaggingMethod);setAiPredefinedTags(account.aiPredefinedTags);+ setAiTagExistingLinks(account.aiTagExistingLinks);setDashboardRecentLinks(account.dashboardRecentLinks);setDashboardPinnedLinks(account.dashboardPinnedLinks);}@@ -143,11 +147,10 @@ export default function Appearance() {].map(({ theme, icon, bgColor, textColor, activeColor }) => (<divkey={theme}- className={`w-full text-center outline-solid outline-neutral-content outline h-20 duration-100 rounded-xl flex items-center justify-center cursor-pointer select-none ${bgColor} ${- localStorage.getItem("theme") === theme- ? `outline-primary ${activeColor}`- : textColor- }`}+ className={`w-full text-center outline-solid outline-neutral-content outline h-20 duration-100 rounded-xl flex items-center justify-center cursor-pointer select-none ${bgColor} ${localStorage.getItem("theme") === theme+ ? `outline-primary ${activeColor}`+ : textColor+ }`}onClick={() => updateSettings({ theme })}><i className={`${icon} text-3xl`}></i>@@ -190,6 +193,14 @@ export default function Appearance() {<div className="divider my-3"></div>+ <div className="mb-3">+ <Checkbox+ label={t("generate_tags_for_existing_links")}+ state={aiTagExistingLinks}+ onClick={() => setAiTagExistingLinks(!aiTagExistingLinks)}+ />+ </div>+<p>{t("ai_tagging_method")}</p><div className="p-3">diff --git a/prisma/migrations/20250129180119_ai_tag_existing_links/migration.sql b/prisma/migrations/20250129180119_ai_tag_existing_links/migration.sqlnew file mode 100644index 00000000..9b84d4dd--- /dev/null+++ b/prisma/migrations/20250129180119_ai_tag_existing_links/migration.sql@@ -0,0 +1,2 @@+-- AlterTable+ALTER TABLE "User" ADD COLUMN "aiTagExistingLinks" BOOLEAN NOT NULL DEFAULT false;diff --git a/prisma/schema.prisma b/prisma/schema.prismaindex 3754eb8f..7da212f2 100644--- a/prisma/schema.prisma+++ b/prisma/schema.prisma@@ -51,6 +51,7 @@ model User {linksRouteTo LinksRouteTo @default(ORIGINAL)aiTaggingMethod AiTaggingMethod @default(DISABLED)aiPredefinedTags String[] @default([])+ aiTagExistingLinks Boolean @default(false)dashboardRecentLinks Boolean @default(true)dashboardPinnedLinks Boolean @default(true)preventDuplicateLinks Boolean @default(false)diff --git a/public/locales/en/common.json b/public/locales/en/common.jsonindex 70ace3c2..f36662d5 100644--- a/public/locales/en/common.json+++ b/public/locales/en/common.json@@ -432,6 +432,7 @@"people_recommendation": "Recommendation (Friend, Family, etc.)","open_all_links": "Open all Links","ai_settings": "AI Settings",+ "generate_tags_for_existing_links": "Generate tags for existing Links","ai_tagging_method": "AI Tagging Method:","based_on_predefined_tags": "Based on predefined Tags","based_on_predefined_tags_desc": "Auto-categorize links to predefined tags based on the content of each link.",diff --git a/scripts/worker.ts b/scripts/worker.tsindex 45e57c1f..b225cc77 100644--- a/scripts/worker.ts+++ b/scripts/worker.ts@@ -1,11 +1,11 @@import "dotenv/config";import { Collection, Link, User } from "@prisma/client";import { prisma } from "../lib/api/db";-import archiveHandler from "../lib/api/archiveHandler";+import archiveHandler, { getBrowserOptions } from "../lib/api/archiveHandler";import Parser from "rss-parser";import { hasPassedLimit } from "../lib/api/verifyCapacity";--const args = process.argv.slice(2).join(" ");+import autoTagLink from "../lib/api/autoTagLink";+import { chromium, devices } from "playwright";const archiveTakeCount = Number(process.env.ARCHIVE_TAKE_COUNT || "") || 5;@@ -107,75 +107,153 @@ async function processBatch() {await Promise.allSettled(processingPromises);}-async function fetchAndProcessRSS() {- const rssSubscriptions = await prisma.rssSubscription.findMany({});- const parser = new Parser();+async function processAITagging() {+ const links = await prisma.link.findMany({+ where: {+ aiTagged: false,+ url: { not: null },+ createdBy: {+ aiTagExistingLinks: true,+ aiTaggingMethod: {+ in: ['GENERATE', 'PREDEFINED']+ }+ }+ },+ include: {+ createdBy: true+ }+ });- rssSubscriptions.forEach(async (rssSubscription) => {- try {- const feed = await parser.parseURL(rssSubscription.url);+ console.log(`Processing ${links.length} links for AI tagging...`);++ await Promise.all(+ links.map(async (link) => {+ let browser = null;+ try {+ if (link.createdBy && link.url) {+ const browserOptions = getBrowserOptions();+ browser = await chromium.launch(browserOptions);++ const context = await browser.newContext({+ ...devices["Desktop Chrome"],+ ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",+ });- if (- rssSubscription.lastBuildDate &&- new Date(rssSubscription.lastBuildDate) < new Date(feed.lastBuildDate)- ) {- console.log(+ const page = await context.newPage();++ await page.goto(link.url, {+ waitUntil: "domcontentloaded",+ timeout: 30000+ });++ const metaDescription = await page.evaluate(() => {+ const description = document.querySelector('meta[name="description"]');+ return description?.getAttribute('content') ?? undefined;+ });++ await autoTagLink(link.createdBy, link.id, metaDescription);++ // Mark the link as processed+ await prisma.link.update({+ where: { id: link.id },+ data: { aiTagged: true }+ });+ }+ } catch (error) {+ console.error("\x1b[34m%s\x1b[0m",- `Processing new RSS feed items for ${rssSubscription.name}`+ `Error auto-tagging link ${link.url}:`,+ error);- const newItems = feed.items.filter((item) => {- const itemPubDate = item.pubDate ? new Date(item.pubDate) : null;- return itemPubDate && itemPubDate > rssSubscription.lastBuildDate!; // We know lastBuildDate is not null here+ // Mark the link as processed even if it failed to prevent endless retries+ await prisma.link.update({+ where: { id: link.id },+ data: { aiTagged: true }});+ } finally {+ if (browser) {+ await browser.close();+ }+ }+ })+ );+}- const hasTooManyLinks = await hasPassedLimit(- rssSubscription.ownerId,- newItems.length- );+async function fetchAndProcessRSS() {+ const rssSubscriptions = await prisma.rssSubscription.findMany({});+ const parser = new Parser();++ await Promise.all(+ rssSubscriptions.map(async (rssSubscription) => {+ try {+ const feed = await parser.parseURL(rssSubscription.url);- if (hasTooManyLinks) {+ if (+ rssSubscription.lastBuildDate &&+ new Date(rssSubscription.lastBuildDate) < new Date(feed.lastBuildDate)+ ) {console.log("\x1b[34m%s\x1b[0m",- `User ${rssSubscription.ownerId} has too many links. Skipping new RSS feed items.`+ `Processing new RSS feed items for ${rssSubscription.name}`);- return;- }- newItems.forEach(async (item) => {- await prisma.link.create({- data: {- name: item.title,- url: item.link,- type: "link",- createdBy: {- connect: {- id: rssSubscription.ownerId,- },- },- collection: {- connect: {- id: rssSubscription.collectionId,- },- },- },+ const newItems = feed.items.filter((item) => {+ const itemPubDate = item.pubDate ? new Date(item.pubDate) : null;+ return itemPubDate && itemPubDate > rssSubscription.lastBuildDate!;});- });- // Update the lastBuildDate in the database- await prisma.rssSubscription.update({- where: { id: rssSubscription.id },- data: { lastBuildDate: new Date(feed.lastBuildDate) },- });+ const hasTooManyLinks = await hasPassedLimit(+ rssSubscription.ownerId,+ newItems.length+ );++ if (hasTooManyLinks) {+ console.log(+ "\x1b[34m%s\x1b[0m",+ `User ${rssSubscription.ownerId} has too many links. Skipping new RSS feed items.`+ );+ return;+ }++ // Create all links concurrently+ await Promise.all(+ newItems.map(async (item) => {+ return prisma.link.create({+ data: {+ name: item.title,+ url: item.link,+ type: "link",+ createdBy: {+ connect: {+ id: rssSubscription.ownerId,+ },+ },+ collection: {+ connect: {+ id: rssSubscription.collectionId,+ },+ },+ },+ });+ })+ );++ // Update the lastBuildDate in the database+ await prisma.rssSubscription.update({+ where: { id: rssSubscription.id },+ data: { lastBuildDate: new Date(feed.lastBuildDate) },+ });+ }+ } catch (error) {+ console.error(+ "\x1b[34m%s\x1b[0m",+ `Error processing RSS feed ${rssSubscription.url}:`,+ error+ );}- } catch (error) {- console.error(- "\x1b[34m%s\x1b[0m",- `Error processing RSS feed ${rssSubscription.url}:`,- error- );- }- });+ })+ );}function delay(sec: number) {@@ -193,6 +271,16 @@ async function startRSSPolling() {}}+const aiTaggingIntervalInSeconds = 10;++async function startAITagging() {+ console.log("\x1b[34m%s\x1b[0m", "Starting AI tagging...");+ while (true) {+ await processAITagging();+ await delay(aiTaggingIntervalInSeconds);+ }+}+const archiveIntervalInSeconds =Number(process.env.ARCHIVE_SCRIPT_INTERVAL) || 10;@@ -207,6 +295,7 @@ async function startArchiveProcessing() {async function init() {console.log("\x1b[34m%s\x1b[0m", "Initializing application...");startRSSPolling();+ startAITagging();startArchiveProcessing();}