Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

Add tag archival settings to the worker

Isaac95f456bbb6f3
diff --git a/lib/api/archiveHandler.ts b/lib/api/archiveHandler.ts
index a6801cb0..0e5ce84d 100644
--- a/lib/api/archiveHandler.ts
+++ b/lib/api/archiveHandler.ts
@@ -1,7 +1,7 @@
import { LaunchOptions, chromium, devices } from "playwright";
import { prisma } from "./db";
import sendToWayback from "./preservationScheme/sendToWayback";
-import { AiTaggingMethod, Collection, Link, User } from "@prisma/client";
+import { AiTaggingMethod } from "@prisma/client";
import fetchHeaders from "./fetchHeaders";
import createFolder from "./storage/createFolder";
import { removeFiles } from "./manageLinkFiles";
@@ -12,16 +12,21 @@ import handleScreenshotAndPdf from "./preservationScheme/handleScreenshotAndPdf"
import imageHandler from "./preservationScheme/imageHandler";
import pdfHandler from "./preservationScheme/pdfHandler";
import autoTagLink from "./autoTagLink";
-
-type LinksAndCollectionAndOwner = Link & {
- collection: Collection & {
- owner: User;
- };
-};
+import { LinkWithCollectionOwnerAndTags } from "../../types/global";
+import { isArchivalTag } from "../../hooks/useArchivalTags";
const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;
-export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
+export interface ArchivalSettings {
+ archiveAsScreenshot: boolean;
+ archiveAsMonolith: boolean;
+ archiveAsPDF: boolean;
+ archiveAsReadable: boolean;
+ archiveAsWaybackMachine: boolean;
+ aiTag: boolean;
+}
+
+export default async function archiveHandler(link: LinkWithCollectionOwnerAndTags) {
if (process.env.DISABLE_PRESERVATION === "true") {
await prisma.link.update({
where: { id: link.id },
@@ -64,6 +69,26 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
const user = link.collection?.owner;
+ const archivalTags = link.tags.filter(isArchivalTag);
+
+ const archivalSettings: ArchivalSettings = archivalTags.length > 0
+ ? {
+ archiveAsScreenshot: archivalTags.some(tag => tag.archiveAsScreenshot),
+ archiveAsMonolith: archivalTags.some(tag => tag.archiveAsMonolith),
+ archiveAsPDF: archivalTags.some(tag => tag.archiveAsPDF),
+ archiveAsReadable: archivalTags.some(tag => tag.archiveAsReadable),
+ archiveAsWaybackMachine: archivalTags.some(tag => tag.archiveAsWaybackMachine),
+ aiTag: archivalTags.some(tag => tag.aiTag),
+ }
+ : {
+ archiveAsScreenshot: user.archiveAsScreenshot,
+ archiveAsMonolith: user.archiveAsMonolith,
+ archiveAsPDF: user.archiveAsPDF,
+ archiveAsReadable: user.archiveAsReadable,
+ archiveAsWaybackMachine: user.archiveAsWaybackMachine,
+ aiTag: user.aiTaggingMethod !== AiTaggingMethod.DISABLED,
+ };
+
try {
await Promise.race([
(async () => {
@@ -73,7 +98,7 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
);
// send to archive.org
- if (user.archiveAsWaybackMachine && link.url) sendToWayback(link.url);
+ if (archivalSettings.archiveAsWaybackMachine && link.url) sendToWayback(link.url);
if (linkType === "image" && !link.image) {
await imageHandler(link, imageExtension); // archive image (jpeg/png)
@@ -92,11 +117,11 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
if (!link.preview) await handleArchivePreview(link, page);
// Readability
- if (user.archiveAsReadable && !link.readable) await handleReadability(content, link);
+ if (archivalSettings.archiveAsReadable && !link.readable) await handleReadability(content, link);
// Auto-tagging
if (
- user.aiTaggingMethod !== AiTaggingMethod.DISABLED &&
+ archivalSettings.aiTag &&
!link.aiTagged &&
process.env.NEXT_PUBLIC_OLLAMA_ENDPOINT_URL
)
@@ -104,13 +129,13 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
// Screenshot/PDF
if (
- (user.archiveAsScreenshot && !link.image) ||
- (user.archiveAsPDF && !link.pdf)
+ (archivalSettings.archiveAsScreenshot && !link.image) ||
+ (archivalSettings.archiveAsPDF && !link.pdf)
)
- await handleScreenshotAndPdf(link, page, user);
+ await handleScreenshotAndPdf(link, page, archivalSettings);
// Monolith
- if (user.archiveAsMonolith && !link.monolith && link.url)
+ if (archivalSettings.archiveAsMonolith && !link.monolith && link.url)
await handleMonolith(link, content);
}
})(),
@@ -135,11 +160,7 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
monolith: !finalLink.monolith ? "unavailable" : undefined,
pdf: !finalLink.pdf ? "unavailable" : undefined,
preview: !finalLink.preview ? "unavailable" : undefined,
- aiTagged:
- user.aiTaggingMethod !== AiTaggingMethod.DISABLED &&
- !finalLink.aiTagged
- ? true
- : undefined,
+ aiTagged: archivalSettings.aiTag && !finalLink.aiTagged || undefined
},
});
else {
diff --git a/lib/api/preservationScheme/handleScreenshotAndPdf.ts b/lib/api/preservationScheme/handleScreenshotAndPdf.ts
index fee5126c..78b68307 100644
--- a/lib/api/preservationScheme/handleScreenshotAndPdf.ts
+++ b/lib/api/preservationScheme/handleScreenshotAndPdf.ts
@@ -1,17 +1,14 @@
-import { Collection, Link, User } from "@prisma/client";
+import { User } from "@prisma/client";
import { Page } from "playwright";
import createFile from "../storage/createFile";
import { prisma } from "../db";
+import { LinkWithCollectionOwnerAndTags } from "../../../types/global";
+import { ArchivalSettings } from "../archiveHandler";
-type LinksAndCollectionAndOwner = Link & {
- collection: Collection & {
- owner: User;
- };
-};
const handleScreenshotAndPdf = async (
- link: LinksAndCollectionAndOwner,
+ link: LinkWithCollectionOwnerAndTags,
page: Page,
- user: User
+ archivalSettings: ArchivalSettings
) => {
await page.evaluate(autoScroll, Number(process.env.AUTOSCROLL_TIMEOUT) || 30);
@@ -22,7 +19,7 @@ const handleScreenshotAndPdf = async (
if (linkExists) {
const processingPromises = [];
- if (user.archiveAsScreenshot && !link.image?.startsWith("archive")) {
+ if (archivalSettings.archiveAsScreenshot && !link.image?.startsWith("archive")) {
processingPromises.push(
page
.screenshot({ fullPage: true, type: "jpeg" })
@@ -42,7 +39,7 @@ const handleScreenshotAndPdf = async (
await prisma.link.update({
where: { id: link.id },
data: {
- image: user.archiveAsScreenshot
+ image: archivalSettings.archiveAsScreenshot
? `archives/${linkExists.collectionId}/${link.id}.jpeg`
: undefined,
},
@@ -56,7 +53,7 @@ const handleScreenshotAndPdf = async (
bottom: process.env.PDF_MARGIN_BOTTOM || "15px",
};
- if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {
+ if (archivalSettings.archiveAsPDF && !link.pdf?.startsWith("archive")) {
processingPromises.push(
page
.pdf({
@@ -82,7 +79,7 @@ const handleScreenshotAndPdf = async (
await prisma.link.update({
where: { id: link.id },
data: {
- pdf: user.archiveAsPDF
+ pdf: archivalSettings.archiveAsPDF
? `archives/${linkExists.collectionId}/${link.id}.pdf`
: undefined,
},
diff --git a/scripts/worker.ts b/scripts/worker.ts
index 45e57c1f..d5921229 100644
--- a/scripts/worker.ts
+++ b/scripts/worker.ts
@@ -1,20 +1,12 @@
import "dotenv/config";
-import { Collection, Link, User } from "@prisma/client";
import { prisma } from "../lib/api/db";
import archiveHandler from "../lib/api/archiveHandler";
import Parser from "rss-parser";
import { hasPassedLimit } from "../lib/api/verifyCapacity";
-
-const args = process.argv.slice(2).join(" ");
+import { LinkWithCollectionOwnerAndTags } from "../types/global";
const archiveTakeCount = Number(process.env.ARCHIVE_TAKE_COUNT || "") || 5;
-type LinksAndCollectionAndOwner = Link & {
- collection: Collection & {
- owner: User;
- };
-};
-
async function processBatch() {
const linksOldToNew = await prisma.link.findMany({
where: {
@@ -42,6 +34,7 @@ async function processBatch() {
owner: true,
},
},
+ tags: true,
},
});
@@ -71,10 +64,11 @@ async function processBatch() {
owner: true,
},
},
+ tags: true,
},
});
- const archiveLink = async (link: LinksAndCollectionAndOwner) => {
+ const archiveLink = async (link: LinkWithCollectionOwnerAndTags) => {
try {
console.log(
"\x1b[34m%s\x1b[0m",
diff --git a/types/global.ts b/types/global.ts
index 1bb87811..63b776bb 100644
--- a/types/global.ts
+++ b/types/global.ts
@@ -58,7 +58,7 @@ export interface AccountSettings extends User {
};
}
-interface LinksIncludingTags extends Link {
+export interface LinksIncludingTags extends Link {
tags: Tag[];
}
@@ -66,6 +66,12 @@ export interface PublicCollectionIncludingLinks extends Collection {
links: LinksIncludingTags[];
}
+export type LinkWithCollectionOwnerAndTags = LinksIncludingTags & {
+ collection: Collection & {
+ owner: User;
+ };
+}
+
export enum ViewMode {
Card = "card",
List = "list",