⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
Add tag archival settings to the worker
Isaac95f456bbb6f3
diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.tsindex a6801cb0..0e5ce84d 100644--- a/lib/api/archiveHandler.ts+++ b/lib/api/archiveHandler.ts@@ -1,7 +1,7 @@import { LaunchOptions, chromium, devices } from "playwright";import { prisma } from "./db";import sendToWayback from "./preservationScheme/sendToWayback";-import { AiTaggingMethod, Collection, Link, User } from "@prisma/client";+import { AiTaggingMethod } from "@prisma/client";import fetchHeaders from "./fetchHeaders";import createFolder from "./storage/createFolder";import { removeFiles } from "./manageLinkFiles";@@ -12,16 +12,21 @@ import handleScreenshotAndPdf from "./preservationScheme/handleScreenshotAndPdf"import imageHandler from "./preservationScheme/imageHandler";import pdfHandler from "./preservationScheme/pdfHandler";import autoTagLink from "./autoTagLink";--type LinksAndCollectionAndOwner = Link & {- collection: Collection & {- owner: User;- };-};+import { LinkWithCollectionOwnerAndTags } from "../../types/global";+import { isArchivalTag } from "../../hooks/useArchivalTags";const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;-export default async function archiveHandler(link: LinksAndCollectionAndOwner) {+export interface ArchivalSettings {+ archiveAsScreenshot: boolean;+ archiveAsMonolith: boolean;+ archiveAsPDF: boolean;+ archiveAsReadable: boolean;+ archiveAsWaybackMachine: boolean;+ aiTag: boolean;+}++export default async function archiveHandler(link: LinkWithCollectionOwnerAndTags) {if (process.env.DISABLE_PRESERVATION === "true") {await prisma.link.update({where: { id: link.id },@@ -64,6 +69,26 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {const user = link.collection?.owner;+ const archivalTags = link.tags.filter(isArchivalTag);++ const archivalSettings: ArchivalSettings = archivalTags.length > 0+ ? {+ archiveAsScreenshot: archivalTags.some(tag => tag.archiveAsScreenshot),+ archiveAsMonolith: archivalTags.some(tag => tag.archiveAsMonolith),+ archiveAsPDF: archivalTags.some(tag => tag.archiveAsPDF),+ archiveAsReadable: archivalTags.some(tag => tag.archiveAsReadable),+ archiveAsWaybackMachine: archivalTags.some(tag => tag.archiveAsWaybackMachine),+ aiTag: archivalTags.some(tag => tag.aiTag),+ }+ : {+ archiveAsScreenshot: user.archiveAsScreenshot,+ archiveAsMonolith: user.archiveAsMonolith,+ archiveAsPDF: user.archiveAsPDF,+ archiveAsReadable: user.archiveAsReadable,+ archiveAsWaybackMachine: user.archiveAsWaybackMachine,+ aiTag: user.aiTaggingMethod !== AiTaggingMethod.DISABLED,+ };+try {await Promise.race([(async () => {@@ -73,7 +98,7 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {);// send to archive.org- if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);+ if (archivalSettings.archiveAsWaybackMachine && link.url) sendToWayback(link.url);if (linkType === "image" && !link.image) {await imageHandler(link, imageExtension); // archive image (jpeg/png)@@ -92,11 +117,11 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {if (!link.preview) await handleArchivePreview(link, page);// Readability- if (user.archiveAsReadable && !link.readable) await handleReadability(content, link);+ if (archivalSettings.archiveAsReadable && !link.readable) await handleReadability(content, link);// Auto-taggingif (- user.aiTaggingMethod !== AiTaggingMethod.DISABLED &&+ archivalSettings.aiTag &&!link.aiTagged &&process.env.NEXT_PUBLIC_OLLAMA_ENDPOINT_URL)@@ -104,13 +129,13 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {// Screenshot/PDFif (- (user.archiveAsScreenshot && !link.image) ||- (user.archiveAsPDF && !link.pdf)+ (archivalSettings.archiveAsScreenshot && !link.image) ||+ (archivalSettings.archiveAsPDF && !link.pdf))- await handleScreenshotAndPdf(link, page, user);+ await handleScreenshotAndPdf(link, page, archivalSettings);// Monolith- if (user.archiveAsMonolith && !link.monolith && link.url)+ if (archivalSettings.archiveAsMonolith && !link.monolith && link.url)await handleMonolith(link, content);}})(),@@ -135,11 +160,7 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {monolith: !finalLink.monolith ? "unavailable" : undefined,pdf: !finalLink.pdf ? "unavailable" : undefined,preview: !finalLink.preview ? "unavailable" : undefined,- aiTagged:- user.aiTaggingMethod !== AiTaggingMethod.DISABLED &&- !finalLink.aiTagged- ? true- : undefined,+ aiTagged: archivalSettings.aiTag && !finalLink.aiTagged || undefined},});else {diff --git a/lib/api/preservationScheme/handleScreenshotAndPdf.ts b/lib/api/preservationScheme/handleScreenshotAndPdf.tsindex fee5126c..78b68307 100644--- a/lib/api/preservationScheme/handleScreenshotAndPdf.ts+++ b/lib/api/preservationScheme/handleScreenshotAndPdf.ts@@ -1,17 +1,14 @@-import { Collection, Link, User } from "@prisma/client";+import { User } from "@prisma/client";import { Page } from "playwright";import createFile from "../storage/createFile";import { prisma } from "../db";+import { LinkWithCollectionOwnerAndTags } from "../../../types/global";+import { ArchivalSettings } from "../archiveHandler";-type LinksAndCollectionAndOwner = Link & {- collection: Collection & {- owner: User;- };-};const handleScreenshotAndPdf = async (- link: LinksAndCollectionAndOwner,+ link: LinkWithCollectionOwnerAndTags,page: Page,- user: User+ archivalSettings: ArchivalSettings) => {await page.evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30);@@ -22,7 +19,7 @@ const handleScreenshotAndPdf = async (if (linkExists) {const processingPromises = [];- if (user.archiveAsScreenshot && !link.image?.startsWith("archive")) {+ if (archivalSettings.archiveAsScreenshot && !link.image?.startsWith("archive")) {processingPromises.push(page.screenshot({ fullPage: true, type: "jpeg" })@@ -42,7 +39,7 @@ const handleScreenshotAndPdf = async (await prisma.link.update({where: { id: link.id },data: {- image: user.archiveAsScreenshot+ image: archivalSettings.archiveAsScreenshot? `archives/${linkExists.collectionId}/${link.id}.jpeg`: undefined,},@@ -56,7 +53,7 @@ const handleScreenshotAndPdf = async (bottom: process.env.PDF_MARGIN_BOTTOM || "15px",};- if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {+ if (archivalSettings.archiveAsPDF && !link.pdf?.startsWith("archive")) {processingPromises.push(page.pdf({@@ -82,7 +79,7 @@ const handleScreenshotAndPdf = async (await prisma.link.update({where: { id: link.id },data: {- pdf: user.archiveAsPDF+ pdf: archivalSettings.archiveAsPDF? `archives/${linkExists.collectionId}/${link.id}.pdf`: undefined,},diff --git a/scripts/worker.ts b/scripts/worker.tsindex 45e57c1f..d5921229 100644--- a/scripts/worker.ts+++ b/scripts/worker.ts@@ -1,20 +1,12 @@import "dotenv/config";-import { Collection, Link, User } from "@prisma/client";import { prisma } from "../lib/api/db";import archiveHandler from "../lib/api/archiveHandler";import Parser from "rss-parser";import { hasPassedLimit } from "../lib/api/verifyCapacity";--const args = process.argv.slice(2).join(" ");+import { LinkWithCollectionOwnerAndTags } from "../types/global";const archiveTakeCount = Number(process.env.ARCHIVE_TAKE_COUNT || "") || 5;-type LinksAndCollectionAndOwner = Link & {- collection: Collection & {- owner: User;- };-};-async function processBatch() {const linksOldToNew = await prisma.link.findMany({where: {@@ -42,6 +34,7 @@ async function processBatch() {owner: true,},},+ tags: true,},});@@ -71,10 +64,11 @@ async function processBatch() {owner: true,},},+ tags: true,},});- const archiveLink = async (link: LinksAndCollectionAndOwner) => {+ const archiveLink = async (link: LinkWithCollectionOwnerAndTags) => {try {console.log("\x1b[34m%s\x1b[0m",diff --git a/types/global.ts b/types/global.tsindex 1bb87811..63b776bb 100644--- a/types/global.ts+++ b/types/global.ts@@ -58,7 +58,7 @@ export interface AccountSettings extends User {};}-interface LinksIncludingTags extends Link {+export interface LinksIncludingTags extends Link {tags: Tag[];}@@ -66,6 +66,12 @@ export interface PublicCollectionIncludingLinks extends Collection {links: LinksIncludingTags[];}+export type LinkWithCollectionOwnerAndTags = LinksIncludingTags & {+ collection: Collection & {+ owner: User;+ };+}+export enum ViewMode {Card = "card",List = "list",