Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

improvements

daniel31x13616efffed23e
diff --git a/.env.sample b/.env.sample
index 5781f1e8..6615c791 100644
--- a/.env.sample
+++ b/.env.sample
@@ -41,7 +41,6 @@ NEXT_PUBLIC_RSS_POLLING_INTERVAL_MINUTES=
RSS_SUBSCRIPTION_LIMIT_PER_USER=
# AI Settings
-AI_TAGGING_SCRIPT_INTERVAL=
NEXT_PUBLIC_OLLAMA_ENDPOINT_URL=
OLLAMA_MODEL=
diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.ts
index 062410d2..724dd59a 100644
--- a/lib/api/archiveHandler.ts
+++ b/lib/api/archiveHandler.ts
@@ -95,6 +95,12 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
const content = await page.content();
+ // Preview
+ if (!link.preview) await handleArchivePreview(link, page);
+
+ // Readability
+ if (!link.readable) await handleReadablility(content, link);
+
// Auto-tagging
if (
user.aiTaggingMethod !== AiTaggingMethod.DISABLED &&
@@ -105,12 +111,6 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
)
await autoTagLink(user, link.id, metaDescription);
- // Preview
- if (!link.preview) await handleArchivePreview(link, page);
-
- // Readability
- if (!link.readable) await handleReadablility(content, link);
-
// Screenshot/PDF
if (
(user.archiveAsScreenshot && !link.image) ||
diff --git a/lib/api/autoTagLink.ts b/lib/api/autoTagLink.ts
index 5973b9a8..0e358dd0 100644
--- a/lib/api/autoTagLink.ts
+++ b/lib/api/autoTagLink.ts
@@ -10,6 +10,7 @@ import { openai } from "@ai-sdk/openai";
import { z } from "zod";
import { anthropic } from "@ai-sdk/anthropic";
import { createOllama } from "ollama-ai-provider";
+import { titleCase } from "../shared/utils";
// Function to concat /api with the base URL properly
const ensureValidURL = (base: string, path: string) =>
@@ -47,16 +48,18 @@ export default async function autoTagLink(
if (!link) return console.log("Link not found for auto tagging.");
- const existingTags = await prisma.tag.findMany({
- select: {
- name: true,
- },
- where: {
- ownerId: user.id,
- },
- });
+ const description =
+ metaDescription ||
+ (link.textContent ? link.textContent?.slice(0, 500) + "..." : undefined);
+
+ if (!description) return;
- console.log("Auto tagging link: ", link.url);
+ console.log(
+ 'Auto tagging "',
+ link.url,
+ '" with the following description: ',
+ description
+ );
let prompt;
@@ -86,17 +89,12 @@ export default async function autoTagLink(
);
}
- const promptText = metaDescription || link.textContent?.slice(0, 500) + "...";
-
- if (!promptText)
- return console.log("No text content to auto tag for link: ", link.url);
-
if (user.aiTaggingMethod === AiTaggingMethod.GENERATE) {
- prompt = generateTagsPrompt(promptText);
+ prompt = generateTagsPrompt(description);
} else if (user.aiTaggingMethod === AiTaggingMethod.EXISTING) {
- prompt = existingTagsPrompt(promptText, existingTagsNames);
+ prompt = existingTagsPrompt(description, existingTagsNames);
} else {
- prompt = predefinedTagsPrompt(promptText, user.aiPredefinedTags);
+ prompt = predefinedTagsPrompt(description, user.aiPredefinedTags);
}
if (
@@ -116,16 +114,21 @@ export default async function autoTagLink(
try {
let tags = object;
- console.log("Tags generated for link: ", link.url, tags);
-
if (!tags || tags.length === 0) {
return;
} else if (user.aiTaggingMethod === AiTaggingMethod.EXISTING) {
tags = tags.filter((tag: string) => existingTagsNames.includes(tag));
} else if (user.aiTaggingMethod === AiTaggingMethod.PREDEFINED) {
tags = tags.filter((tag: string) => user.aiPredefinedTags.includes(tag));
+ } else if (user.aiTaggingMethod === AiTaggingMethod.GENERATE) {
+ tags = tags.map((tag: string) =>
+ // I was thinking of doing something like this instead: tag.length > 3 ? titleCase(tag.toLowerCase()) : tag
+ titleCase(tag.toLowerCase())
+ );
}
+ console.log("Tags for link:", link.url, "=>", tags);
+
if (tags.length > 5) {
tags = tags.slice(0, 5);
}
@@ -137,12 +140,12 @@ export default async function autoTagLink(
connectOrCreate: tags.map((tag: string) => ({
where: {
name_ownerId: {
- name: tag.trim().slice(0, 100),
+ name: tag.trim().slice(0, 50),
ownerId: user.id,
},
},
create: {
- name: tag.trim().slice(0, 100),
+ name: tag.trim().slice(0, 50),
owner: {
connect: {
id: user.id,
diff --git a/lib/api/prompts.ts b/lib/api/prompts.ts
index 2c7ba345..d6c0b035 100644
--- a/lib/api/prompts.ts
+++ b/lib/api/prompts.ts
@@ -2,7 +2,6 @@ export const generateTagsPrompt = (text: string) => `
You are a Bookmark Manager that should extract relevant tags from the following text, here are the rules:
- The final output should be only an array of tags.
- The tags should be in the language of the text.
-- Every word in a tag must start with a capital letter.
- The maximum number of tags is 5.
- Each tag should be maximum one to two words.
- If there are no tags, return an empty array.
@@ -18,7 +17,6 @@ Predefined tags: ${tags.join(", ")}.
Here are the rules:
- The final output should be only an array of tags.
- The tags should be in the language of the text.
-- Every word in a tag must start with a capital letter.
- The maximum number of tags is 5.
- Each tag should be maximum one to two words.
- If there are no tags, return an empty array.
@@ -34,7 +32,6 @@ The existing tags are sorted from most used to least used: ${tags.join(", ")}.
Here are the rules:
- The final output should be only an array of tags.
- The tags should be in the language of the text.
-- Every word in a tag must start with a capital letter.
- The maximum number of tags is 5.
- Each tag should be maximum one to two words.
- If there are no tags, return an empty array.
diff --git a/lib/shared/utils.ts b/lib/shared/utils.ts
new file mode 100644
index 00000000..804fa7b3
--- /dev/null
+++ b/lib/shared/utils.ts
@@ -0,0 +1,8 @@
+export function titleCase(str: string) {
+ var splitStr = str.toLowerCase().split(" ");
+ for (var i = 0; i < splitStr.length; i++) {
+ splitStr[i] =
+ splitStr[i].charAt(0).toUpperCase() + splitStr[i].substring(1);
+ }
+ return splitStr.join(" ");
+}
diff --git a/prisma/seed.js b/prisma/seed.js
index 91765ec8..40c3d364 100644
--- a/prisma/seed.js
+++ b/prisma/seed.js
@@ -279,6 +279,7 @@ async function main() {
name: link.name,
url: link.url,
collectionId: projectAlphaCollection.id,
+ createdById: user2.id,
},
});
}
@@ -306,6 +307,7 @@ async function main() {
name: link.name,
url: link.url,
collectionId: collection.id,
+ createdById: user1.id,
},
});
}
diff --git a/scripts/worker.ts b/scripts/worker.ts
index b229c019..bacbb369 100644
--- a/scripts/worker.ts
+++ b/scripts/worker.ts
@@ -1,11 +1,9 @@
import "dotenv/config";
import { Collection, Link, User } from "@prisma/client";
import { prisma } from "../lib/api/db";
-import archiveHandler, { getBrowserOptions } from "../lib/api/archiveHandler";
+import archiveHandler from "../lib/api/archiveHandler";
import Parser from "rss-parser";
import { hasPassedLimit } from "../lib/api/verifyCapacity";
-import autoTagLink from "../lib/api/autoTagLink";
-import { chromium, devices } from "playwright";
const archiveTakeCount = Number(process.env.ARCHIVE_TAKE_COUNT || "") || 5;
@@ -32,6 +30,15 @@ async function processBatch() {
{
monolith: null,
},
+ {
+ createdBy: {
+ aiTagExistingLinks: true,
+ NOT: {
+ aiTaggingMethod: "DISABLED",
+ },
+ },
+ aiTagged: false,
+ },
],
},
take: archiveTakeCount,
@@ -61,6 +68,15 @@ async function processBatch() {
{
monolith: null,
},
+ {
+ createdBy: {
+ aiTagExistingLinks: true,
+ NOT: {
+ aiTaggingMethod: "DISABLED",
+ },
+ },
+ aiTagged: false,
+ },
],
},
take: archiveTakeCount,
@@ -107,79 +123,6 @@ async function processBatch() {
await Promise.allSettled(processingPromises);
}
-async function processAITagging() {
- const links = await prisma.link.findMany({
- where: {
- aiTagged: false,
- url: { not: null },
- createdBy: {
- aiTagExistingLinks: true,
- aiTaggingMethod: {
- in: ["GENERATE", "EXISTING", "PREDEFINED"],
- },
- },
- },
- include: {
- createdBy: true,
- },
- });
-
- await Promise.all(
- links.map(async (link) => {
- let browser = null;
- try {
- if (link.createdBy && link.url) {
- const browserOptions = getBrowserOptions();
- browser = await chromium.launch(browserOptions);
-
- const context = await browser.newContext({
- ...devices["Desktop Chrome"],
- ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",
- });
-
- const page = await context.newPage();
-
- await page.goto(link.url, {
- waitUntil: "domcontentloaded",
- timeout: 30000,
- });
-
- const metaDescription = await page.evaluate(() => {
- const description = document.querySelector(
- 'meta[name="description"]'
- );
- return description?.getAttribute("content") ?? undefined;
- });
-
- await autoTagLink(link.createdBy, link.id, metaDescription);
-
- // Mark the link as processed
- await prisma.link.update({
- where: { id: link.id },
- data: { aiTagged: true },
- });
- }
- } catch (error) {
- console.error(
- "\x1b[34m%s\x1b[0m",
- `Error auto-tagging link ${link.url}:`,
- error
- );
-
- // Mark the link as processed even if it failed to prevent endless retries
- await prisma.link.update({
- where: { id: link.id },
- data: { aiTagged: true },
- });
- } finally {
- if (browser) {
- await browser.close();
- }
- }
- })
- );
-}
-
async function fetchAndProcessRSS() {
const rssSubscriptions = await prisma.rssSubscription.findMany({});
const parser = new Parser();
@@ -271,17 +214,6 @@ async function startRSSPolling() {
}
}
-const aiTaggingIntervalInSeconds =
- Number(process.env.AI_TAGGING_SCRIPT_INTERVAL) || 30;
-
-async function startAITagging() {
- console.log("\x1b[34m%s\x1b[0m", "Starting AI tagging...");
- while (true) {
- await processAITagging();
- await delay(aiTaggingIntervalInSeconds);
- }
-}
-
const archiveIntervalInSeconds =
Number(process.env.ARCHIVE_SCRIPT_INTERVAL) || 10;
@@ -296,7 +228,6 @@ async function startArchiveProcessing() {
async function init() {
console.log("\x1b[34m%s\x1b[0m", "Initializing application...");
startRSSPolling();
- startAITagging();
startArchiveProcessing();
}