⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
improvements
daniel31x13616efffed23e
diff --git a/.env.sample b/.env.sampleindex 5781f1e8..6615c791 100644--- a/.env.sample+++ b/.env.sample@@ -41,7 +41,6 @@ NEXT_PUBLIC_RSS_POLLING_INTERVAL_MINUTES=RSS_SUBSCRIPTION_LIMIT_PER_USER=# AI Settings-AI_TAGGING_SCRIPT_INTERVAL=NEXT_PUBLIC_OLLAMA_ENDPOINT_URL=OLLAMA_MODEL=diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.tsindex 062410d2..724dd59a 100644--- a/lib/api/archiveHandler.ts+++ b/lib/api/archiveHandler.ts@@ -95,6 +95,12 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {const content = await page.content();+ // Preview+ if (!link.preview) await handleArchivePreview(link, page);++ // Readability+ if (!link.readable) await handleReadablility(content, link);+// Auto-taggingif (user.aiTaggingMethod !== AiTaggingMethod.DISABLED &&@@ -105,12 +111,6 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {)await autoTagLink(user, link.id, metaDescription);- // Preview- if (!link.preview) await handleArchivePreview(link, page);-- // Readability- if (!link.readable) await handleReadablility(content, link);-// Screenshot/PDFif ((user.archiveAsScreenshot && !link.image) ||diff --git a/lib/api/autoTagLink.ts b/lib/api/autoTagLink.tsindex 5973b9a8..0e358dd0 100644--- a/lib/api/autoTagLink.ts+++ b/lib/api/autoTagLink.ts@@ -10,6 +10,7 @@ import { openai } from "@ai-sdk/openai";import { z } from "zod";import { anthropic } from "@ai-sdk/anthropic";import { createOllama } from "ollama-ai-provider";+import { titleCase } from "../shared/utils";// Function to concat /api with the base URL properlyconst ensureValidURL = (base: string, path: string) =>@@ -47,16 +48,18 @@ export default async function autoTagLink(if (!link) return console.log("Link not found for auto tagging.");- const existingTags = await prisma.tag.findMany({- select: {- name: true,- },- where: {- ownerId: user.id,- },- });+ const description =+ metaDescription ||+ (link.textContent ? link.textContent?.slice(0, 500) + "..." : undefined);++ if (!description) return;- console.log("Auto tagging link: ", link.url);+ console.log(+ 'Auto tagging "',+ link.url,+ '" with the following description: ',+ description+ );let prompt;@@ -86,17 +89,12 @@ export default async function autoTagLink();}- const promptText = metaDescription || link.textContent?.slice(0, 500) + "...";-- if (!promptText)- return console.log("No text content to auto tag for link: ", link.url);-if (user.aiTaggingMethod === AiTaggingMethod.GENERATE) {- prompt = generateTagsPrompt(promptText);+ prompt = generateTagsPrompt(description);} else if (user.aiTaggingMethod === AiTaggingMethod.EXISTING) {- prompt = existingTagsPrompt(promptText, existingTagsNames);+ prompt = existingTagsPrompt(description, existingTagsNames);} else {- prompt = predefinedTagsPrompt(promptText, user.aiPredefinedTags);+ prompt = predefinedTagsPrompt(description, user.aiPredefinedTags);}if (@@ -116,16 +114,21 @@ export default async function autoTagLink(try {let tags = object;- console.log("Tags generated for link: ", link.url, tags);-if (!tags || tags.length === 0) {return;} else if (user.aiTaggingMethod === AiTaggingMethod.EXISTING) {tags = tags.filter((tag: string) => existingTagsNames.includes(tag));} else if (user.aiTaggingMethod === AiTaggingMethod.PREDEFINED) {tags = tags.filter((tag: string) => user.aiPredefinedTags.includes(tag));+ } else if (user.aiTaggingMethod === AiTaggingMethod.GENERATE) {+ tags = tags.map((tag: string) =>+ // I was thinking of doing something like this instead: tag.length > 3 ? titleCase(tag.toLowerCase()) : tag+ titleCase(tag.toLowerCase())+ );}+ console.log("Tags for link:", link.url, "=>", tags);+if (tags.length > 5) {tags = tags.slice(0, 5);}@@ -137,12 +140,12 @@ export default async function autoTagLink(connectOrCreate: tags.map((tag: string) => ({where: {name_ownerId: {- name: tag.trim().slice(0, 100),+ name: tag.trim().slice(0, 50),ownerId: user.id,},},create: {- name: tag.trim().slice(0, 100),+ name: tag.trim().slice(0, 50),owner: {connect: {id: user.id,diff --git a/lib/api/prompts.ts b/lib/api/prompts.tsindex 2c7ba345..d6c0b035 100644--- a/lib/api/prompts.ts+++ b/lib/api/prompts.ts@@ -2,7 +2,6 @@ export const generateTagsPrompt = (text: string) => `You are a Bookmark Manager that should extract relevant tags from the following text, here are the rules:- The final output should be only an array of tags.- The tags should be in the language of the text.-- Every word in a tag must start with a capital letter.- The maximum number of tags is 5.- Each tag should be maximum one to two words.- If there are no tags, return an empty array.@@ -18,7 +17,6 @@ Predefined tags: ${tags.join(", ")}.Here are the rules:- The final output should be only an array of tags.- The tags should be in the language of the text.-- Every word in a tag must start with a capital letter.- The maximum number of tags is 5.- Each tag should be maximum one to two words.- If there are no tags, return an empty array.@@ -34,7 +32,6 @@ The existing tags are sorted from most used to least used: ${tags.join(", ")}.Here are the rules:- The final output should be only an array of tags.- The tags should be in the language of the text.-- Every word in a tag must start with a capital letter.- The maximum number of tags is 5.- Each tag should be maximum one to two words.- If there are no tags, return an empty array.diff --git a/lib/shared/utils.ts b/lib/shared/utils.tsnew file mode 100644index 00000000..804fa7b3--- /dev/null+++ b/lib/shared/utils.ts@@ -0,0 +1,8 @@+export function titleCase(str: string) {+ var splitStr = str.toLowerCase().split(" ");+ for (var i = 0; i < splitStr.length; i++) {+ splitStr[i] =+ splitStr[i].charAt(0).toUpperCase() + splitStr[i].substring(1);+ }+ return splitStr.join(" ");+}diff --git a/prisma/seed.js b/prisma/seed.jsindex 91765ec8..40c3d364 100644--- a/prisma/seed.js+++ b/prisma/seed.js@@ -279,6 +279,7 @@ async function main() {name: link.name,url: link.url,collectionId: projectAlphaCollection.id,+ createdById: user2.id,},});}@@ -306,6 +307,7 @@ async function main() {name: link.name,url: link.url,collectionId: collection.id,+ createdById: user1.id,},});}diff --git a/scripts/worker.ts b/scripts/worker.tsindex b229c019..bacbb369 100644--- a/scripts/worker.ts+++ b/scripts/worker.ts@@ -1,11 +1,9 @@import "dotenv/config";import { Collection, Link, User } from "@prisma/client";import { prisma } from "../lib/api/db";-import archiveHandler, { getBrowserOptions } from "../lib/api/archiveHandler";+import archiveHandler from "../lib/api/archiveHandler";import Parser from "rss-parser";import { hasPassedLimit } from "../lib/api/verifyCapacity";-import autoTagLink from "../lib/api/autoTagLink";-import { chromium, devices } from "playwright";const archiveTakeCount = Number(process.env.ARCHIVE_TAKE_COUNT || "") || 5;@@ -32,6 +30,15 @@ async function processBatch() {{monolith: null,},+ {+ createdBy: {+ aiTagExistingLinks: true,+ NOT: {+ aiTaggingMethod: "DISABLED",+ },+ },+ aiTagged: false,+ },],},take: archiveTakeCount,@@ -61,6 +68,15 @@ async function processBatch() {{monolith: null,},+ {+ createdBy: {+ aiTagExistingLinks: true,+ NOT: {+ aiTaggingMethod: "DISABLED",+ },+ },+ aiTagged: false,+ },],},take: archiveTakeCount,@@ -107,79 +123,6 @@ async function processBatch() {await Promise.allSettled(processingPromises);}-async function processAITagging() {- const links = await prisma.link.findMany({- where: {- aiTagged: false,- url: { not: null },- createdBy: {- aiTagExistingLinks: true,- aiTaggingMethod: {- in: ["GENERATE", "EXISTING", "PREDEFINED"],- },- },- },- include: {- createdBy: true,- },- });-- await Promise.all(- links.map(async (link) => {- let browser = null;- try {- if (link.createdBy && link.url) {- const browserOptions = getBrowserOptions();- browser = await chromium.launch(browserOptions);-- const context = await browser.newContext({- ...devices["Desktop Chrome"],- ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",- });-- const page = await context.newPage();-- await page.goto(link.url, {- waitUntil: "domcontentloaded",- timeout: 30000,- });-- const metaDescription = await page.evaluate(() => {- const description = document.querySelector(- 'meta[name="description"]'- );- return description?.getAttribute("content") ?? undefined;- });-- await autoTagLink(link.createdBy, link.id, metaDescription);-- // Mark the link as processed- await prisma.link.update({- where: { id: link.id },- data: { aiTagged: true },- });- }- } catch (error) {- console.error(- "\x1b[34m%s\x1b[0m",- `Error auto-tagging link ${link.url}:`,- error- );-- // Mark the link as processed even if it failed to prevent endless retries- await prisma.link.update({- where: { id: link.id },- data: { aiTagged: true },- });- } finally {- if (browser) {- await browser.close();- }- }- })- );-}-async function fetchAndProcessRSS() {const rssSubscriptions = await prisma.rssSubscription.findMany({});const parser = new Parser();@@ -271,17 +214,6 @@ async function startRSSPolling() {}}-const aiTaggingIntervalInSeconds =- Number(process.env.AI_TAGGING_SCRIPT_INTERVAL) || 30;--async function startAITagging() {- console.log("\x1b[34m%s\x1b[0m", "Starting AI tagging...");- while (true) {- await processAITagging();- await delay(aiTaggingIntervalInSeconds);- }-}-const archiveIntervalInSeconds =Number(process.env.ARCHIVE_SCRIPT_INTERVAL) || 10;@@ -296,7 +228,6 @@ async function startArchiveProcessing() {async function init() {console.log("\x1b[34m%s\x1b[0m", "Initializing application...");startRSSPolling();- startAITagging();startArchiveProcessing();}