Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

fix: fix banner extraction of singlefile archives

Mohamed Bassemba6adfcef341
diff --git a/apps/workers/package.json b/apps/workers/package.json
index f7037fe8..9376fac9 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -18,6 +18,7 @@
"archiver": "^7.0.1",
"async-mutex": "^0.4.1",
"better-sqlite3": "^11.3.0",
+ "data-uri-to-buffer": "^4.0.1",
"dompurify": "^3.2.4",
"dotenv": "^16.4.1",
"drizzle-orm": "^0.45.2",
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts
index 1e4bcfda..91d1fff9 100644
--- a/apps/workers/workers/crawlerWorker.ts
+++ b/apps/workers/workers/crawlerWorker.ts
@@ -7,6 +7,8 @@ import { Transform } from "stream";
import { pipeline } from "stream/promises";
import { PlaywrightBlocker } from "@ghostery/adblocker-playwright";
import { Mutex } from "async-mutex";
+import { dataUriToBuffer } from "data-uri-to-buffer";
+import type { MimeBuffer } from "data-uri-to-buffer";
import { and, eq } from "drizzle-orm";
import { execa } from "execa";
import { exitAbortController } from "exit";
@@ -1366,6 +1368,82 @@ async function storePdf(
);
}
+/**
+ * SingleFile / precrawled archives inline images as `data:` URIs, so the image
+ * URL metascraper extracts can be a base64 blob rather than something fetchable
+ * over the network. Decode it locally and store it as an asset instead of
+ * failing on the unsupported protocol.
+ */
+async function storeDataUriAsset(
+ url: string,
+ userId: string,
+ jobId: string,
+ fileType: string,
+) {
+ const maxBytes = serverConfig.maxAssetSizeMb * 1024 * 1024;
+
+ // Guardrail 1: cap the size *before* decoding into memory. base64 encodes 3
+ // bytes per 4 chars, so a URI longer than 4/3 the limit (plus header slack)
+ // must decode past it; the exact check below handles the rest.
+ if (url.length > Math.ceil((maxBytes * 4) / 3) + 1024) {
+ logger.warn(
+ `[Crawler][${jobId}] Skipping data URI ${fileType}: encoded size (${url.length} chars) exceeds maximum allowed size of ${serverConfig.maxAssetSizeMb}MB`,
+ );
+ return null;
+ }
+
+ let asset: MimeBuffer;
+ try {
+ asset = dataUriToBuffer(url);
+ } catch (e) {
+ logger.error(
+ `[Crawler][${jobId}] Failed to decode data URI ${fileType}: ${e}`,
+ );
+ return null;
+ }
+
+ // Guardrail 2: never trust the declared mediatype for anything but the raster
+ // image types we actually serve as banners. Rejects svg/html/etc.;
+ // saveAsset's SUPPORTED_ASSET_TYPES check is the backstop.
+ const contentType = normalizeContentType(asset.type);
+ if (!contentType || !IMAGE_ASSET_TYPES.has(contentType)) {
+ logger.warn(
+ `[Crawler][${jobId}] Skipping data URI ${fileType} with unsupported content type: ${contentType}`,
+ );
+ return null;
+ }
+
+ if (asset.byteLength > maxBytes) {
+ logger.warn(
+ `[Crawler][${jobId}] Skipping data URI ${fileType}: decoded size (${asset.byteLength} bytes) exceeds maximum allowed size of ${serverConfig.maxAssetSizeMb}MB`,
+ );
+ return null;
+ }
+
+ const { data: quotaApproved, error: quotaError } = await tryCatch(
+ QuotaService.checkStorageQuota(db, userId, asset.byteLength),
+ );
+ if (quotaError) {
+ logger.warn(
+ `[Crawler][${jobId}] Skipping data URI ${fileType} storage due to quota exceeded: ${quotaError.message}`,
+ );
+ return null;
+ }
+
+ const assetId = newAssetId();
+ await saveAsset({
+ userId,
+ assetId,
+ metadata: { contentType },
+ asset,
+ quotaApproved,
+ });
+ logger.info(
+ `[Crawler][${jobId}] Stored data URI ${fileType} as assetId: ${assetId} (${asset.byteLength} bytes)`,
+ );
+ return { assetId, userId, contentType, size: asset.byteLength };
+}
+
async function downloadAndStoreFile(
url: string,
userId: string,
@@ -1389,6 +1467,9 @@ async function downloadAndStoreFile(
async () => {
let assetPath: string | undefined;
try {
+ if (url.startsWith("data:")) {
+ return await storeDataUriAsset(url, userId, jobId, fileType);
+ }
logger.info(
`[Crawler][${jobId}] Downloading ${fileType} from "${truncateUrl(url)}"`,
);
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 238752eb..4bcea791 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -898,6 +898,9 @@ importers:
better-sqlite3:
specifier: ^11.3.0
version: 11.3.0
+ data-uri-to-buffer:
+ specifier: ^4.0.1
+ version: 4.0.1
dompurify:
specifier: ^3.2.4
version: 3.2.6
@@ -16509,6 +16512,7 @@ packages:
resolution: {integrity: sha512-ks6Vjr/jEw0P1gmOVwutM3B7fWxoWBL2KRDb1JfqGVawBmO5UsvmWOQFGHBPl5yxYz4eERr19E6L7NMv+Fej4w==}
engines: {node: ^18 || >=20}
+ deprecated: unmaintained
hasBin: true
peerDependencies:
typescript: ^5.0.0