A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
fix: fix banner extraction of singlefile archives
Mohamed Bassemba6adfcef341
diff --git a/apps/workers/package.json b/apps/workers/package.jsonindex f7037fe8..9376fac9 100644--- a/apps/workers/package.json+++ b/apps/workers/package.json@@ -18,6 +18,7 @@"archiver": "^7.0.1","async-mutex": "^0.4.1","better-sqlite3": "^11.3.0",+ "data-uri-to-buffer": "^4.0.1","dompurify": "^3.2.4","dotenv": "^16.4.1","drizzle-orm": "^0.45.2",diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.tsindex 1e4bcfda..91d1fff9 100644--- a/apps/workers/workers/crawlerWorker.ts+++ b/apps/workers/workers/crawlerWorker.ts@@ -7,6 +7,8 @@ import { Transform } from "stream";import { pipeline } from "stream/promises";import { PlaywrightBlocker } from "@ghostery/adblocker-playwright";import { Mutex } from "async-mutex";+import { dataUriToBuffer } from "data-uri-to-buffer";+import type { MimeBuffer } from "data-uri-to-buffer";import { and, eq } from "drizzle-orm";import { execa } from "execa";import { exitAbortController } from "exit";@@ -1366,6 +1368,82 @@ async function storePdf();}+/**+ * SingleFile / precrawled archives inline images as `data:` URIs, so the image+ * URL metascraper extracts can be a base64 blob rather than something fetchable+ * over the network. Decode it locally and store it as an asset instead of+ * failing on the unsupported protocol.+ */+async function storeDataUriAsset(+ url: string,+ userId: string,+ jobId: string,+ fileType: string,+) {+ const maxBytes = serverConfig.maxAssetSizeMb * 1024 * 1024;++ // Guardrail 1: cap the size *before* decoding into memory. base64 encodes 3+ // bytes per 4 chars, so a URI longer than 4/3 the limit (plus header slack)+ // must decode past it; the exact check below handles the rest.+ if (url.length > Math.ceil((maxBytes * 4) / 3) + 1024) {+ logger.warn(+ `[Crawler][${jobId}] Skipping data URI ${fileType}: encoded size (${url.length} chars) exceeds maximum allowed size of ${serverConfig.maxAssetSizeMb}MB`,+ );+ return null;+ }++ let asset: MimeBuffer;+ try {+ asset = dataUriToBuffer(url);+ } catch (e) {+ logger.error(+ `[Crawler][${jobId}] Failed to decode data URI ${fileType}: ${e}`,+ );+ return null;+ }++ // Guardrail 2: never trust the declared mediatype for anything but the raster+ // image types we actually serve as banners. Rejects svg/html/etc.;+ // saveAsset's SUPPORTED_ASSET_TYPES check is the backstop.+ const contentType = normalizeContentType(asset.type);+ if (!contentType || !IMAGE_ASSET_TYPES.has(contentType)) {+ logger.warn(+ `[Crawler][${jobId}] Skipping data URI ${fileType} with unsupported content type: ${contentType}`,+ );+ return null;+ }++ if (asset.byteLength > maxBytes) {+ logger.warn(+ `[Crawler][${jobId}] Skipping data URI ${fileType}: decoded size (${asset.byteLength} bytes) exceeds maximum allowed size of ${serverConfig.maxAssetSizeMb}MB`,+ );+ return null;+ }++ const { data: quotaApproved, error: quotaError } = await tryCatch(+ QuotaService.checkStorageQuota(db, userId, asset.byteLength),+ );+ if (quotaError) {+ logger.warn(+ `[Crawler][${jobId}] Skipping data URI ${fileType} storage due to quota exceeded: ${quotaError.message}`,+ );+ return null;+ }++ const assetId = newAssetId();+ await saveAsset({+ userId,+ assetId,+ metadata: { contentType },+ asset,+ quotaApproved,+ });+ logger.info(+ `[Crawler][${jobId}] Stored data URI ${fileType} as assetId: ${assetId} (${asset.byteLength} bytes)`,+ );+ return { assetId, userId, contentType, size: asset.byteLength };+}+async function downloadAndStoreFile(url: string,userId: string,@@ -1389,6 +1467,9 @@ async function downloadAndStoreFile(async () => {let assetPath: string | undefined;try {+ if (url.startsWith("data:")) {+ return await storeDataUriAsset(url, userId, jobId, fileType);+ }logger.info(`[Crawler][${jobId}] Downloading ${fileType} from "${truncateUrl(url)}"`,);diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 238752eb..4bcea791 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -898,6 +898,9 @@ importers:better-sqlite3:specifier: ^11.3.0version: 11.3.0+ data-uri-to-buffer:+ specifier: ^4.0.1+ version: 4.0.1dompurify:specifier: ^3.2.4version: 3.2.6@@ -16509,6 +16512,7 @@ packages:resolution: {integrity: sha512-ks6Vjr/jEw0P1gmOVwutM3B7fWxoWBL2KRDb1JfqGVawBmO5UsvmWOQFGHBPl5yxYz4eERr19E6L7NMv+Fej4w==}engines: {node: ^18 || >=20}+ deprecated: unmaintainedhasBin: truepeerDependencies:typescript: ^5.0.0