Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

auto tag existing links & use meta description

Isaac9d8d5f0fa00f
diff --git a/.env.sample b/.env.sample
index 6615c791..5781f1e8 100644
--- a/.env.sample
+++ b/.env.sample
@@ -41,6 +41,7 @@ NEXT_PUBLIC_RSS_POLLING_INTERVAL_MINUTES=
RSS_SUBSCRIPTION_LIMIT_PER_USER=
# AI Settings
+AI_TAGGING_SCRIPT_INTERVAL=
NEXT_PUBLIC_OLLAMA_ENDPOINT_URL=
OLLAMA_MODEL=
diff --git a/components/Preservation/Preservation.tsx b/components/Preservation/Preservation.tsx
index 83148815..0048c3d6 100644
--- a/components/Preservation/Preservation.tsx
+++ b/components/Preservation/Preservation.tsx
@@ -114,7 +114,7 @@ export default function Preservation({
className={clsx(
"bg-base-200",
!isExpanded &&
- `mx-auto relative p-2 h-full overflow-auto rounded-md sm:rounded-none`,
+ `mx-auto relative p-2 h-full overflow-auto rounded-md sm:rounded-none`,
isExpanded && "fixed inset-0 w-screen h-screen z-[9999]"
)}
style={{
@@ -129,7 +129,7 @@ export default function Preservation({
className={clsx(
"py-1 px-2 cursor-pointer duration-100 rounded-lg font-semibold text-center flex justify-between items-center w-6",
format === ArchivedFormat.readability &&
- "bg-primary bg-opacity-50"
+ "bg-primary bg-opacity-50"
)}
onClick={() => setFormat(ArchivedFormat.readability)}
>
@@ -143,7 +143,7 @@ export default function Preservation({
className={clsx(
"py-1 px-2 cursor-pointer duration-100 rounded-lg font-semibold text-center flex justify-between items-center w-6",
format === ArchivedFormat.monolith &&
- "bg-primary bg-opacity-50"
+ "bg-primary bg-opacity-50"
)}
onClick={() => setFormat(ArchivedFormat.monolith)}
>
@@ -157,9 +157,9 @@ export default function Preservation({
className={clsx(
"py-1 px-2 cursor-pointer duration-100 rounded-lg font-semibold text-center flex justify-between items-center w-6",
format ===
- (link?.image?.endsWith("png")
- ? ArchivedFormat.png
- : ArchivedFormat.jpeg) && "bg-primary bg-opacity-50"
+ (link?.image?.endsWith("png")
+ ? ArchivedFormat.png
+ : ArchivedFormat.jpeg) && "bg-primary bg-opacity-50"
)}
onClick={() =>
setFormat(
@@ -279,29 +279,29 @@ const RenderFormat = ({
const style = isExpanded
? {
- position: "fixed" as const,
- top: 0,
- left: 0,
- width: "100vw",
- height: "100vh",
- zIndex: 9999,
- transitionProperty:
- delayPassed && !shouldReduceMotion
- ? "width, height, top, left"
- : "none",
- }
+ position: "fixed" as const,
+ top: 0,
+ left: 0,
+ width: "100vw",
+ height: "100vh",
+ zIndex: 9999,
+ transitionProperty:
+ delayPassed && !shouldReduceMotion
+ ? "width, height, top, left"
+ : "none",
+ }
: {
- zIndex: 40,
- position: "absolute" as const,
- top: (containerRect?.top ?? 0) + window.scrollY + topPadding,
- left: (containerRect?.left ?? 0) + window.scrollX,
- width: containerRect?.width ?? 0,
- height: containerRect?.height ? containerRect.height - topPadding : 0,
- transitionProperty:
- delayPassed && !shouldReduceMotion
- ? "width, height, top, left"
- : "none",
- };
+ zIndex: 40,
+ position: "absolute" as const,
+ top: (containerRect?.top ?? 0) + window.scrollY + topPadding,
+ left: (containerRect?.left ?? 0) + window.scrollX,
+ width: containerRect?.width ?? 0,
+ height: containerRect?.height ? containerRect.height - topPadding : 0,
+ transitionProperty:
+ delayPassed && !shouldReduceMotion
+ ? "width, height, top, left"
+ : "none",
+ };
return ReactDOM.createPortal(
<div
diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.ts
index bb3ed339..e25ad66f 100644
--- a/lib/api/archiveHandler.ts
+++ b/lib/api/archiveHandler.ts
@@ -86,13 +86,12 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
await page.goto(link.url, { waitUntil: "domcontentloaded" });
- const content = await page.content();
-
- // Preview
- if (!link.preview) await handleArchivePreview(link, page);
+ const metaDescription = await page.evaluate(() => {
+ const description = document.querySelector('meta[name="description"]');
+ return description?.getAttribute('content') ?? undefined;
+ });
- // Readability
- if (!link.readable) await handleReadablility(content, link);
+ const content = await page.content();
// Auto-tagging
if (
@@ -104,7 +103,13 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
process.env.ANTHROPIC_API_KEY
)
)
- await autoTagLink(user, link.id);
+ await autoTagLink(user, link.id, metaDescription);
+
+ // Preview
+ if (!link.preview) await handleArchivePreview(link, page);
+
+ // Readability
+ if (!link.readable) await handleReadablility(content, link);
// Screenshot/PDF
if (
@@ -189,7 +194,7 @@ async function determineLinkType(
}
// Construct browser launch options based on environment variables.
-function getBrowserOptions(): LaunchOptions {
+export function getBrowserOptions(): LaunchOptions {
let browserOptions: LaunchOptions = {};
if (process.env.PROXY) {
diff --git a/lib/api/autoTagLink.ts b/lib/api/autoTagLink.ts
index 40a5fdae..26aa056c 100644
--- a/lib/api/autoTagLink.ts
+++ b/lib/api/autoTagLink.ts
@@ -26,7 +26,7 @@ const getAIModel = () => {
throw new Error("No AI provider configured");
}
-export default async function autoTagLink(user: User, linkId: number) {
+export default async function autoTagLink(user: User, linkId: number, metaDescription: string | undefined) {
const link = await prisma.link.findUnique({
where: { id: linkId },
});
@@ -37,15 +37,15 @@ export default async function autoTagLink(user: User, linkId: number) {
let prompt;
- const textContent = link.textContent?.slice(0, 500) + "...";
+ const promptText = metaDescription || link.textContent?.slice(0, 500) + "...";
- if (!link.textContent)
+ if (!promptText)
return console.log("No text content to auto tag for link: ", link.url);
if (user.aiTaggingMethod === AiTaggingMethod.GENERATE) {
- prompt = generateTagsPrompt(textContent);
+ prompt = generateTagsPrompt(promptText);
} else {
- prompt = predefinedTagsPrompt(textContent, user.aiPredefinedTags);
+ prompt = predefinedTagsPrompt(promptText, user.aiPredefinedTags);
}
if (
diff --git a/lib/api/controllers/links/postLink.ts b/lib/api/controllers/links/postLink.ts
index 76e0eb85..5cc885b0 100644
--- a/lib/api/controllers/links/postLink.ts
+++ b/lib/api/controllers/links/postLink.ts
@@ -16,9 +16,8 @@ export default async function postLink(
if (!dataValidation.success) {
return {
- response: `Error: ${
- dataValidation.error.issues[0].message
- } [${dataValidation.error.issues[0].path.join(", ")}]`,
+ response: `Error: ${dataValidation.error.issues[0].message
+ } [${dataValidation.error.issues[0].path.join(", ")}]`,
status: 400,
};
}
@@ -134,9 +133,8 @@ export default async function postLink(
where: { id: newLink.id },
data: {
image: link.image
- ? `archives/${newLink.collectionId}/${newLink.id}.${
- link.image === "png" ? "png" : "jpeg"
- }`
+ ? `archives/${newLink.collectionId}/${newLink.id}.${link.image === "png" ? "png" : "jpeg"
+ }`
: undefined,
},
});
diff --git a/lib/api/controllers/users/userId/updateUserById.ts b/lib/api/controllers/users/userId/updateUserById.ts
index 7f1e0892..69393d25 100644
--- a/lib/api/controllers/users/userId/updateUserById.ts
+++ b/lib/api/controllers/users/userId/updateUserById.ts
@@ -19,9 +19,8 @@ export default async function updateUserById(
if (!dataValidation.success) {
return {
- response: `Error: ${
- dataValidation.error.issues[0].message
- } [${dataValidation.error.issues[0].path.join(", ")}]`,
+ response: `Error: ${dataValidation.error.issues[0].message
+ } [${dataValidation.error.issues[0].path.join(", ")}]`,
status: 400,
};
}
@@ -33,18 +32,18 @@ export default async function updateUserById(
id: { not: userId },
OR: emailEnabled
? [
- {
- username: data.username.toLowerCase(),
- },
- {
- email: data.email?.toLowerCase(),
- },
- ]
+ {
+ username: data.username.toLowerCase(),
+ },
+ {
+ email: data.email?.toLowerCase(),
+ },
+ ]
: [
- {
- username: data.username.toLowerCase(),
- },
- ],
+ {
+ username: data.username.toLowerCase(),
+ },
+ ],
},
});
@@ -203,6 +202,7 @@ export default async function updateUserById(
),
aiTaggingMethod: data.aiTaggingMethod,
aiPredefinedTags: data.aiPredefinedTags,
+ aiTagExistingLinks: data.aiTagExistingLinks,
locale: i18n.locales.includes(data.locale || "") ? data.locale : "en",
archiveAsScreenshot: data.archiveAsScreenshot,
archiveAsMonolith: data.archiveAsMonolith,
diff --git a/lib/shared/fetchTitleAndHeaders.ts b/lib/shared/fetchTitleAndHeaders.ts
index a7779646..fcfee319 100644
--- a/lib/shared/fetchTitleAndHeaders.ts
+++ b/lib/shared/fetchTitleAndHeaders.ts
@@ -42,7 +42,7 @@ export default async function fetchTitleAndHeaders(url: string) {
// regular expression to find the <title> tag
let match = text.match(/<title.*>([^<]*)<\/title>/);
- const title = match[1] || "";
+ const title = match?.[1] || "";
const headers = (response as Response)?.headers || null;
return { title, headers };
diff --git a/lib/shared/schemaValidation.ts b/lib/shared/schemaValidation.ts
index d549c9f5..7f80699d 100644
--- a/lib/shared/schemaValidation.ts
+++ b/lib/shared/schemaValidation.ts
@@ -41,12 +41,12 @@ export const PostUserSchema = () => {
username: emailEnabled
? z.string().optional()
: z
- .string()
- .trim()
- .toLowerCase()
- .min(3)
- .max(50)
- .regex(/^[a-z0-9_-]{3,50}$/),
+ .string()
+ .trim()
+ .toLowerCase()
+ .min(3)
+ .max(50)
+ .regex(/^[a-z0-9_-]{3,50}$/),
invite: z.boolean().optional(),
});
};
@@ -79,6 +79,7 @@ export const UpdateUserSchema = () => {
dashboardRecentLinks: z.boolean().optional(),
aiTaggingMethod: z.nativeEnum(AiTaggingMethod).optional(),
aiPredefinedTags: z.array(z.string().max(20).trim()).max(20).optional(),
+ aiTagExistingLinks: z.boolean().optional(),
locale: z.string().max(20).optional(),
isPrivate: z.boolean().optional(),
preventDuplicateLinks: z.boolean().optional(),
diff --git a/pages/settings/preference.tsx b/pages/settings/preference.tsx
index 7fd4208b..25849358 100644
--- a/pages/settings/preference.tsx
+++ b/pages/settings/preference.tsx
@@ -44,6 +44,7 @@ export default function Appearance() {
account.aiTaggingMethod
);
const [aiPredefinedTags, setAiPredefinedTags] = useState<string[]>();
+ const [aiTagExistingLinks, setAiTagExistingLinks] = useState<boolean>(account.aiTagExistingLinks);
const { data: config } = useConfig();
@@ -58,6 +59,7 @@ export default function Appearance() {
preventDuplicateLinks,
aiTaggingMethod,
aiPredefinedTags,
+ aiTagExistingLinks,
dashboardRecentLinks,
dashboardPinnedLinks,
});
@@ -71,6 +73,7 @@ export default function Appearance() {
preventDuplicateLinks,
aiTaggingMethod,
aiPredefinedTags,
+ aiTagExistingLinks,
dashboardRecentLinks,
dashboardPinnedLinks,
]);
@@ -89,6 +92,7 @@ export default function Appearance() {
setPreventDuplicateLinks(account.preventDuplicateLinks);
setAiTaggingMethod(account.aiTaggingMethod);
setAiPredefinedTags(account.aiPredefinedTags);
+ setAiTagExistingLinks(account.aiTagExistingLinks);
setDashboardRecentLinks(account.dashboardRecentLinks);
setDashboardPinnedLinks(account.dashboardPinnedLinks);
}
@@ -143,11 +147,10 @@ export default function Appearance() {
].map(({ theme, icon, bgColor, textColor, activeColor }) => (
<div
key={theme}
- className={`w-full text-center outline-solid outline-neutral-content outline h-20 duration-100 rounded-xl flex items-center justify-center cursor-pointer select-none ${bgColor} ${
- localStorage.getItem("theme") === theme
- ? `outline-primary ${activeColor}`
- : textColor
- }`}
+ className={`w-full text-center outline-solid outline-neutral-content outline h-20 duration-100 rounded-xl flex items-center justify-center cursor-pointer select-none ${bgColor} ${localStorage.getItem("theme") === theme
+ ? `outline-primary ${activeColor}`
+ : textColor
+ }`}
onClick={() => updateSettings({ theme })}
>
<i className={`${icon} text-3xl`}></i>
@@ -190,6 +193,14 @@ export default function Appearance() {
<div className="divider my-3"></div>
+ <div className="mb-3">
+ <Checkbox
+ label={t("generate_tags_for_existing_links")}
+ state={aiTagExistingLinks}
+ onClick={() => setAiTagExistingLinks(!aiTagExistingLinks)}
+ />
+ </div>
+
<p>{t("ai_tagging_method")}</p>
<div className="p-3">
diff --git a/prisma/migrations/20250129180119_ai_tag_existing_links/migration.sql b/prisma/migrations/20250129180119_ai_tag_existing_links/migration.sql
new file mode 100644
index 00000000..9b84d4dd
--- /dev/null
+++ b/prisma/migrations/20250129180119_ai_tag_existing_links/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "User" ADD COLUMN "aiTagExistingLinks" BOOLEAN NOT NULL DEFAULT false;
diff --git a/prisma/schema.prisma b/prisma/schema.prisma
index 3754eb8f..7da212f2 100644
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -51,6 +51,7 @@ model User {
linksRouteTo LinksRouteTo @default(ORIGINAL)
aiTaggingMethod AiTaggingMethod @default(DISABLED)
aiPredefinedTags String[] @default([])
+ aiTagExistingLinks Boolean @default(false)
dashboardRecentLinks Boolean @default(true)
dashboardPinnedLinks Boolean @default(true)
preventDuplicateLinks Boolean @default(false)
diff --git a/public/locales/en/common.json b/public/locales/en/common.json
index 70ace3c2..f36662d5 100644
--- a/public/locales/en/common.json
+++ b/public/locales/en/common.json
@@ -432,6 +432,7 @@
"people_recommendation": "Recommendation (Friend, Family, etc.)",
"open_all_links": "Open all Links",
"ai_settings": "AI Settings",
+ "generate_tags_for_existing_links": "Generate tags for existing Links",
"ai_tagging_method": "AI Tagging Method:",
"based_on_predefined_tags": "Based on predefined Tags",
"based_on_predefined_tags_desc": "Auto-categorize links to predefined tags based on the content of each link.",
diff --git a/scripts/worker.ts b/scripts/worker.ts
index 45e57c1f..b225cc77 100644
--- a/scripts/worker.ts
+++ b/scripts/worker.ts
@@ -1,11 +1,11 @@
import "dotenv/config";
import { Collection, Link, User } from "@prisma/client";
import { prisma } from "../lib/api/db";
-import archiveHandler from "../lib/api/archiveHandler";
+import archiveHandler, { getBrowserOptions } from "../lib/api/archiveHandler";
import Parser from "rss-parser";
import { hasPassedLimit } from "../lib/api/verifyCapacity";
-
-const args = process.argv.slice(2).join(" ");
+import autoTagLink from "../lib/api/autoTagLink";
+import { chromium, devices } from "playwright";
const archiveTakeCount = Number(process.env.ARCHIVE_TAKE_COUNT || "") || 5;
@@ -107,75 +107,153 @@ async function processBatch() {
await Promise.allSettled(processingPromises);
}
-async function fetchAndProcessRSS() {
- const rssSubscriptions = await prisma.rssSubscription.findMany({});
- const parser = new Parser();
+async function processAITagging() {
+ const links = await prisma.link.findMany({
+ where: {
+ aiTagged: false,
+ url: { not: null },
+ createdBy: {
+ aiTagExistingLinks: true,
+ aiTaggingMethod: {
+ in: ['GENERATE', 'PREDEFINED']
+ }
+ }
+ },
+ include: {
+ createdBy: true
+ }
+ });
- rssSubscriptions.forEach(async (rssSubscription) => {
- try {
- const feed = await parser.parseURL(rssSubscription.url);
+ console.log(`Processing ${links.length} links for AI tagging...`);
+
+ await Promise.all(
+ links.map(async (link) => {
+ let browser = null;
+ try {
+ if (link.createdBy && link.url) {
+ const browserOptions = getBrowserOptions();
+ browser = await chromium.launch(browserOptions);
+
+ const context = await browser.newContext({
+ ...devices["Desktop Chrome"],
+ ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",
+ });
- if (
- rssSubscription.lastBuildDate &&
- new Date(rssSubscription.lastBuildDate) < new Date(feed.lastBuildDate)
- ) {
- console.log(
+ const page = await context.newPage();
+
+ await page.goto(link.url, {
+ waitUntil: "domcontentloaded",
+ timeout: 30000
+ });
+
+ const metaDescription = await page.evaluate(() => {
+ const description = document.querySelector('meta[name="description"]');
+ return description?.getAttribute('content') ?? undefined;
+ });
+
+ await autoTagLink(link.createdBy, link.id, metaDescription);
+
+ // Mark the link as processed
+ await prisma.link.update({
+ where: { id: link.id },
+ data: { aiTagged: true }
+ });
+ }
+ } catch (error) {
+ console.error(
"\x1b[34m%s\x1b[0m",
- `Processing new RSS feed items for ${rssSubscription.name}`
+ `Error auto-tagging link ${link.url}:`,
+ error
);
- const newItems = feed.items.filter((item) => {
- const itemPubDate = item.pubDate ? new Date(item.pubDate) : null;
- return itemPubDate && itemPubDate > rssSubscription.lastBuildDate!; // We know lastBuildDate is not null here
+ // Mark the link as processed even if it failed to prevent endless retries
+ await prisma.link.update({
+ where: { id: link.id },
+ data: { aiTagged: true }
});
+ } finally {
+ if (browser) {
+ await browser.close();
+ }
+ }
+ })
+ );
+}
- const hasTooManyLinks = await hasPassedLimit(
- rssSubscription.ownerId,
- newItems.length
- );
+async function fetchAndProcessRSS() {
+ const rssSubscriptions = await prisma.rssSubscription.findMany({});
+ const parser = new Parser();
+
+ await Promise.all(
+ rssSubscriptions.map(async (rssSubscription) => {
+ try {
+ const feed = await parser.parseURL(rssSubscription.url);
- if (hasTooManyLinks) {
+ if (
+ rssSubscription.lastBuildDate &&
+ new Date(rssSubscription.lastBuildDate) < new Date(feed.lastBuildDate)
+ ) {
console.log(
"\x1b[34m%s\x1b[0m",
- `User ${rssSubscription.ownerId} has too many links. Skipping new RSS feed items.`
+ `Processing new RSS feed items for ${rssSubscription.name}`
);
- return;
- }
- newItems.forEach(async (item) => {
- await prisma.link.create({
- data: {
- name: item.title,
- url: item.link,
- type: "link",
- createdBy: {
- connect: {
- id: rssSubscription.ownerId,
- },
- },
- collection: {
- connect: {
- id: rssSubscription.collectionId,
- },
- },
- },
+ const newItems = feed.items.filter((item) => {
+ const itemPubDate = item.pubDate ? new Date(item.pubDate) : null;
+ return itemPubDate && itemPubDate > rssSubscription.lastBuildDate!;
});
- });
- // Update the lastBuildDate in the database
- await prisma.rssSubscription.update({
- where: { id: rssSubscription.id },
- data: { lastBuildDate: new Date(feed.lastBuildDate) },
- });
+ const hasTooManyLinks = await hasPassedLimit(
+ rssSubscription.ownerId,
+ newItems.length
+ );
+
+ if (hasTooManyLinks) {
+ console.log(
+ "\x1b[34m%s\x1b[0m",
+ `User ${rssSubscription.ownerId} has too many links. Skipping new RSS feed items.`
+ );
+ return;
+ }
+
+ // Create all links concurrently
+ await Promise.all(
+ newItems.map(async (item) => {
+ return prisma.link.create({
+ data: {
+ name: item.title,
+ url: item.link,
+ type: "link",
+ createdBy: {
+ connect: {
+ id: rssSubscription.ownerId,
+ },
+ },
+ collection: {
+ connect: {
+ id: rssSubscription.collectionId,
+ },
+ },
+ },
+ });
+ })
+ );
+
+ // Update the lastBuildDate in the database
+ await prisma.rssSubscription.update({
+ where: { id: rssSubscription.id },
+ data: { lastBuildDate: new Date(feed.lastBuildDate) },
+ });
+ }
+ } catch (error) {
+ console.error(
+ "\x1b[34m%s\x1b[0m",
+ `Error processing RSS feed ${rssSubscription.url}:`,
+ error
+ );
}
- } catch (error) {
- console.error(
- "\x1b[34m%s\x1b[0m",
- `Error processing RSS feed ${rssSubscription.url}:`,
- error
- );
- }
- });
+ })
+ );
}
function delay(sec: number) {
@@ -193,6 +271,16 @@ async function startRSSPolling() {
}
}
+const aiTaggingIntervalInSeconds = 10;
+
+async function startAITagging() {
+ console.log("\x1b[34m%s\x1b[0m", "Starting AI tagging...");
+ while (true) {
+ await processAITagging();
+ await delay(aiTaggingIntervalInSeconds);
+ }
+}
+
const archiveIntervalInSeconds =
Number(process.env.ARCHIVE_SCRIPT_INTERVAL) || 10;
@@ -207,6 +295,7 @@ async function startArchiveProcessing() {
async function init() {
console.log("\x1b[34m%s\x1b[0m", "Initializing application...");
startRSSPolling();
+ startAITagging();
startArchiveProcessing();
}