Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

feat: add extra metadata parse in the probe fetch (#2932)

Mohamed Bassem84399288e4af
diff --git a/apps/workers/scripts/parseHtmlSubprocess.ts b/apps/workers/scripts/parseHtmlSubprocess.ts
index 60f2b6fc..bc9c4a3d 100644
--- a/apps/workers/scripts/parseHtmlSubprocess.ts
+++ b/apps/workers/scripts/parseHtmlSubprocess.ts
@@ -160,7 +160,7 @@ async function main() {
const input = parseSubprocessInputSchema.parse(
JSON.parse(Buffer.concat(chunks).toString()),
);
- const { htmlContent, url, jobId } = input;
+ const { htmlContent, url, jobId, metadataOnly } = input;
logger.info(
`[Crawler][${jobId}] Will attempt to extract metadata from page ...`,
@@ -177,7 +177,7 @@ async function main() {
// Conditionally run readability (skip if metascraper already provided readable content, e.g. Reddit plugin)
let readableContent: { content: string } | null = null;
- if (meta.readableContentHtml) {
+ if (!metadataOnly && meta.readableContentHtml) {
// Sanitize plugin-provided HTML through DOMPurify (the extractReadableContent
// path already does this, but the direct-content path was missing it).
const purifyWindow = new JSDOM("").window;
@@ -190,7 +190,7 @@ async function main() {
}
}
- if (!readableContent) {
+ if (!metadataOnly && !readableContent) {
logger.info(
`[Crawler][${jobId}] Will attempt to extract readable content ...`,
);
diff --git a/apps/workers/workerUtils.ts b/apps/workers/workerUtils.ts
index 77b50009..999a5790 100644
--- a/apps/workers/workerUtils.ts
+++ b/apps/workers/workerUtils.ts
@@ -33,6 +33,7 @@ export async function getBookmarkDetails(bookmarkId: string) {
userId: bookmark.userId,
createdAt: bookmark.createdAt,
crawledAt: bookmark.link.crawledAt,
+ probeMetadataAt: bookmark.link.probeMetadataAt,
screenshotAssetId: bookmark.assets.find(
(a) => a.assetType == AssetTypes.LINK_SCREENSHOT,
)?.id,
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts
index 933ce1d3..bbd28b91 100644
--- a/apps/workers/workers/crawlerWorker.ts
+++ b/apps/workers/workers/crawlerWorker.ts
@@ -9,7 +9,7 @@ import { PlaywrightBlocker } from "@ghostery/adblocker-playwright";
import { Mutex } from "async-mutex";
import { dataUriToBuffer } from "data-uri-to-buffer";
import type { MimeBuffer } from "data-uri-to-buffer";
-import { and, eq } from "drizzle-orm";
+import { and, eq, sql } from "drizzle-orm";
import { execa } from "execa";
import { exitAbortController } from "exit";
import {
@@ -34,7 +34,13 @@ import {
} from "playwright";
import { chromium } from "playwright-extra";
import StealthPlugin from "puppeteer-extra-plugin-stealth";
-import { abortRace, abortRaceResolve, raceWith, timeoutRace } from "utils";
+import {
+ abortRace,
+ abortRaceResolve,
+ raceWith,
+ timeoutRace,
+ timeoutRejectRace,
+} from "utils";
import { withWorkerTracing, withWorkerEventLog } from "workerTracing";
import { getBookmarkDetails, updateAsset } from "workerUtils";
import { z } from "zod";
@@ -97,6 +103,10 @@ import {
parseSubprocessErrorSchema,
parseSubprocessOutputSchema,
} from "./utils/parseHtmlSubprocessIpc";
+import {
+ isLikelyChallengePage,
+ resolveMetadata,
+} from "./utils/metadataResolver";
const tracer = getTracer("@karakeep/workers");
@@ -1206,6 +1216,7 @@ async function runParseSubprocess(
url: string,
jobId: string,
abortSignal: AbortSignal,
+ opts?: { metadataOnly?: boolean },
): Promise<{
metadata: ParseSubprocessOutput["metadata"];
readableContent: { content: string } | null;
@@ -1229,7 +1240,12 @@ async function runParseSubprocess(
const timeoutMs = serverConfig.crawler.parseTimeoutSec * 1000;
const result = await execa({
- input: JSON.stringify({ htmlContent, url, jobId }),
+ input: JSON.stringify({
+ htmlContent,
+ url,
+ jobId,
+ metadataOnly: opts?.metadataOnly,
+ }),
cancelSignal: abortSignal,
timeout: timeoutMs,
reject: false,
@@ -1738,15 +1754,51 @@ async function archiveWebpage(
);
}
-async function getContentType(
+// Cap how much of the probed page we buffer for metadata extraction.
+// Preview metadata lives in <head>, so a couple of MB is plenty.
+const PROBE_MAX_BODY_BYTES = 2 * 1024 * 1024;
+
+// The content-type routing decision needs only the response headers, so it
+// keeps the tight timeout the probe always had. The body download (for
+// metadata extraction) runs alongside the browser crawl, so it gets a more
+// generous overall budget.
+const PROBE_HEADERS_TIMEOUT_MS = 5_000;
+const PROBE_TIMEOUT_MS = 30_000;
+
+const PROBE_HTML_CONTENT_TYPES = new Set<string>([
+ ASSET_TYPES.TEXT_HTML,
+ "application/xhtml+xml",
+]);
+
+interface UrlProbeResult {
+ contentType: string | null;
+ /**
+ * Resolves with the page's preview metadata (or null). The extraction runs
+ * in the background so it can overlap with the browser crawl — await it
+ * only when the metadata is actually needed. Never rejects.
+ */
+ metadata: Promise<ParseSubprocessOutput["metadata"] | null>;
+}
+
+/**
+ * Probes the URL with a plain GET to determine its content type. When the
+ * response is an HTML page, the body is also parsed (via the parse subprocess
+ * in metadata-only mode) so callers get the page's preview metadata without
+ * waiting for the full browser render. The returned promise resolves once the
+ * content type is known; the metadata extraction keeps running in the
+ * background behind `metadata`. Extraction is best-effort and never affects
+ * the returned content type.
+ */
+async function getContentTypeAndMetadata(
url: string,
jobId: string,
abortSignal: AbortSignal,
runProxy: RunProxyConfig,
-): Promise<string | null> {
+ opts?: { skipMetadataExtraction?: boolean },
+): Promise<UrlProbeResult> {
return await withSpan(
tracer,
- "crawlerWorker.getContentType",
+ "crawlerWorker.getContentTypeAndMetadata",
{
attributes: {
"bookmark.url": url,
@@ -1755,40 +1807,237 @@ async function getContentType(
},
},
async () => {
+ // The request-level signal uses the long budget (it also governs the
+ // body read); the wait for the headers is raced separately against the
+ // short timeout so a dead host can't delay the routing decision.
+ const probeAbort = new AbortController();
+ let response;
try {
logger.info(
`[Crawler][${jobId}] Attempting to determine the content-type for the url ${truncateUrl(url)}`,
);
- const response = await fetchWithProxy(
- url,
- {
- method: "GET",
- signal: AbortSignal.any([AbortSignal.timeout(5000), abortSignal]),
- },
- runProxy,
- );
- setSpanAttributes({
- "crawler.getContentType.statusCode": response.status,
- });
- const rawContentType = response.headers.get("content-type");
- const contentType = normalizeContentType(rawContentType);
- setSpanAttributes({
- "crawler.contentType": contentType ?? undefined,
- });
- logger.info(
- `[Crawler][${jobId}] Content-type for the url ${truncateUrl(url)} is "${contentType}"`,
+ response = await raceWith(
+ fetchWithProxy(
+ url,
+ {
+ method: "GET",
+ signal: AbortSignal.any([
+ AbortSignal.timeout(PROBE_TIMEOUT_MS),
+ abortSignal,
+ probeAbort.signal,
+ ]),
+ size: PROBE_MAX_BODY_BYTES,
+ headers: serverConfig.crawler.preflightUserAgent
+ ? { "User-Agent": serverConfig.crawler.preflightUserAgent }
+ : undefined,
+ },
+ runProxy,
+ ),
+ timeoutRejectRace(
+ PROBE_HEADERS_TIMEOUT_MS,
+ `Timed out after ${PROBE_HEADERS_TIMEOUT_MS}ms waiting for the response headers`,
+ ),
);
- return contentType;
} catch (e) {
+ // Stop the underlying fetch if it's still in flight (header timeout).
+ probeAbort.abort();
logger.error(
`[Crawler][${jobId}] Failed to determine the content-type for the url ${truncateUrl(url)}: ${e}`,
);
- return null;
+ return { contentType: null, metadata: Promise.resolve(null) };
+ }
+ setSpanAttributes({
+ "crawler.getContentType.statusCode": response.status,
+ });
+ const rawContentType = response.headers.get("content-type");
+ const contentType = normalizeContentType(rawContentType);
+ setSpanAttributes({
+ "crawler.contentType": contentType ?? undefined,
+ });
+ logger.info(
+ `[Crawler][${jobId}] Content-type for the url ${truncateUrl(url)} is "${contentType}"`,
+ );
+
+ if (!contentType || !PROBE_HTML_CONTENT_TYPES.has(contentType)) {
+ return { contentType, metadata: Promise.resolve(null) };
+ }
+
+ // A previous run already extracted and stored this page's metadata; the
+ // probe was only needed for the content-type routing decision.
+ if (opts?.skipMetadataExtraction) {
+ logger.info(
+ `[Crawler][${jobId}] Skipping metadata extraction from the content-type probe for the url ${truncateUrl(url)} as it was already fetched by a previous run`,
+ );
+ addLogFields<"crawlerWorker.run">({
+ "crawler.probe.metadata": "reused_stored",
+ });
+ return { contentType, metadata: Promise.resolve(null) };
+ }
+
+ // A blocked/retryable status usually means a challenge or error page
+ // whose metadata would be junk — don't extract it.
+ if (shouldRetryCrawlStatusCode(response.status)) {
+ logger.info(
+ `[Crawler][${jobId}] Skipping metadata extraction from the content-type probe for the url ${truncateUrl(url)} due to status code ${response.status}`,
+ );
+ addLogFields<"crawlerWorker.run">({
+ "crawler.probe.metadata": "blocked_status",
+ });
+ return { contentType, metadata: Promise.resolve(null) };
}
+
+ // The response is an HTML page: parse its metadata from the body we've
+ // already fetched. This is deliberately NOT awaited so it runs alongside
+ // the browser crawl. It must never reject (callers may await it late or
+ // not at all) and any failure here must not lose the content type, as
+ // that would regress the asset-vs-webpage routing decision.
+ const metadata = (async () => {
+ try {
+ const htmlContent = await response.text();
+ const { metadata: parsedMetadata } = await runParseSubprocess(
+ htmlContent,
+ response.url,
+ jobId,
+ abortSignal,
+ { metadataOnly: true },
+ );
+ if (
+ isLikelyChallengePage({ title: parsedMetadata.title, htmlContent })
+ ) {
+ logger.info(
+ `[Crawler][${jobId}] The content-type probe response for the url ${truncateUrl(url)} looks like a bot-challenge page; ignoring its metadata`,
+ );
+ addLogFields<"crawlerWorker.run">({
+ "crawler.probe.metadata": "challenge_page",
+ });
+ return null;
+ }
+ logger.info(
+ `[Crawler][${jobId}] Extracted page metadata from the content-type probe for the url ${truncateUrl(url)}`,
+ );
+ addLogFields<"crawlerWorker.run">({
+ "crawler.probe.metadata": "extracted",
+ });
+ return parsedMetadata;
+ } catch (e) {
+ logger.warn(
+ `[Crawler][${jobId}] Failed to extract page metadata from the content-type probe for the url ${truncateUrl(url)}: ${e}`,
+ );
+ addLogFields<"crawlerWorker.run">({
+ "crawler.probe.metadata": "failed",
+ });
+ return null;
+ }
+ })();
+ return { contentType, metadata };
},
);
}
+/**
+ * Writes the preview metadata extracted by the pre-crawl probe so the bookmark
+ * gets a title/thumbnail before the (slow) browser render finishes.
+ *
+ * The write is strictly fill-only: each field is wrapped in
+ * COALESCE(NULLIF(existing, ''), new) so it can only populate fields that are
+ * currently empty and never override an existing value. This matters because
+ * the probe extraction runs detached from the crawl attempt's lifecycle — if
+ * an attempt fails early, this write can fire after a *retry* (possibly on
+ * another worker) has already stored its final metadata, and it must not
+ * clobber it. The post-render metadata write refines these values within the
+ * owning attempt.
+ */
+async function writeProbeMetadata(
+ bookmarkId: string,
+ metadata: ParseSubprocessOutput["metadata"],
+ jobId: string,
+) {
+ // Don't store data URIs as they're not valid URLs and are usually quite large
+ const image =
+ metadata.image && !metadata.image.startsWith("data:")
+ ? metadata.image
+ : null;
+ if (!metadata.title && !metadata.description && !image && !metadata.logo) {
+ return;
+ }
+ await db
+ .update(bookmarkLinks)
+ .set({
+ ...(metadata.title
+ ? {
+ title: sql`COALESCE(NULLIF(${bookmarkLinks.title}, ''), ${metadata.title})`,
+ }
+ : {}),
+ ...(metadata.description
+ ? {
+ description: sql`COALESCE(NULLIF(${bookmarkLinks.description}, ''), ${metadata.description})`,
+ }
+ : {}),
+ ...(image
+ ? {
+ imageUrl: sql`COALESCE(NULLIF(${bookmarkLinks.imageUrl}, ''), ${image})`,
+ }
+ : {}),
+ ...(metadata.logo
+ ? {
+ favicon: sql`COALESCE(NULLIF(${bookmarkLinks.favicon}, ''), ${metadata.logo})`,
+ }
+ : {}),
+ // Also record that the probe metadata has been fetched and stored, so
+ // crawl retries can skip re-fetching it (see loadStoredProbeMetadata).
+ probeMetadataAt: new Date(),
+ })
+ .where(eq(bookmarkLinks.id, bookmarkId));
+ logger.info(
+ `[Crawler][${jobId}] Wrote early metadata from the content-type probe.`,
+ );
+}
+
+/**
+ * Reloads previously stored probe metadata from the bookmark row. Used on
+ * crawl retries (where `probeMetadataAt` shows a probe already succeeded)
+ * instead of re-fetching and re-parsing the page, so the blocked-render merge
+ * protection keeps working without the extra fetch. Never rejects.
+ */
+async function loadStoredProbeMetadata(
+ bookmarkId: string,
+ jobId: string,
+): Promise<ParseSubprocessOutput["metadata"] | null> {
+ const { data: row, error } = await tryCatch(
+ db.query.bookmarkLinks.findFirst({
+ where: eq(bookmarkLinks.id, bookmarkId),
+ columns: {
+ title: true,
+ description: true,
+ imageUrl: true,
+ favicon: true,
+ author: true,
+ publisher: true,
+ datePublished: true,
+ dateModified: true,
+ },
+ }),
+ );
+ if (error || !row) {
+ if (error) {
+ logger.warn(
+ `[Crawler][${jobId}] Failed to load stored probe metadata: ${error}`,
+ );
+ }
+ return null;
+ }
+ return {
+ title: row.title,
+ description: row.description,
+ image: row.imageUrl,
+ logo: row.favicon,
+ author: row.author,
+ publisher: row.publisher,
+ datePublished: row.datePublished?.toISOString() ?? null,
+ dateModified: row.dateModified?.toISOString() ?? null,
+ };
+}
+
/**
* Downloads the asset from the URL and transforms the linkBookmark to an assetBookmark
* @param url the url the user provided
@@ -1971,6 +2220,7 @@ async function crawlAndParseUrl(
numRetriesLeft: number,
abortSignal: AbortSignal,
runProxy: RunProxyConfig,
+ probeMetadataPromise: Promise<ParseSubprocessOutput["metadata"] | null>,
) {
const sanitizedProxyUrl = redactUrlCredentials(
runProxy.httpsProxy ?? runProxy.httpProxy ?? "",
@@ -2060,10 +2310,33 @@ async function crawlAndParseUrl(
);
}
- const { metadata: meta, readableContent: parsedReadableContent } =
+ const { metadata: renderMeta, readableContent: parsedReadableContent } =
await runParseSubprocess(htmlContent, browserUrl, jobId, abortSignal);
abortSignal.throwIfAborted();
+ // The probe metadata extraction has been running alongside the crawl;
+ // this is the point where it's needed. The promise never rejects and
+ // all its underlying work is time-bounded.
+ const probeMetadata = await probeMetadataPromise;
+ abortSignal.throwIfAborted();
+
+ // On the last retry attempt the crawl proceeds despite a blocked status
+ // code, and some bot walls serve their challenge page with a 200; in
+ // both cases don't let that page's metadata override clean values from
+ // the preflight probe.
+ const renderBlocked =
+ shouldRetryCrawlStatusCode(statusCode) ||
+ isLikelyChallengePage({ title: renderMeta.title, htmlContent });
+ addLogFields<"crawlerWorker.run">({
+ "crawler.render_blocked": renderBlocked,
+ });
+ if (renderBlocked && probeMetadata) {
+ logger.info(
+ `[Crawler][${jobId}] The rendered page looks blocked (status ${statusCode}); preferring preflight probe metadata.`,
+ );
+ }
+ const meta = resolveMetadata(renderMeta, probeMetadata, renderBlocked);
+
const parseDate = (date: string | null | undefined) => {
if (!date) {
return null;
@@ -2347,6 +2620,7 @@ async function runCrawler(
fullPageArchiveAssetId: oldFullPageArchiveAssetId,
contentAssetId: oldContentAssetId,
precrawledArchiveAssetId,
+ probeMetadataAt,
} = await getBookmarkDetails(bookmarkId);
await checkDomainRateLimit(url, jobId);
@@ -2371,9 +2645,17 @@ async function runCrawler(
`[Crawler][${jobId}] Skipped fetching content-type for the url ${url} as precrawledArchiveAssetId exists`,
);
}
- const contentType = precrawledArchiveAssetId
- ? ASSET_TYPES.TEXT_HTML
- : await getContentType(url, jobId, job.abortSignal, runProxy);
+ // Retry runs re-probe for the content type, but if a previous run already
+ // extracted and stored the probe metadata (probeMetadataAt), don't re-fetch
+ // it — reload it from the bookmark row instead.
+ const reuseStoredProbeMetadata =
+ job.runNumber > 0 && probeMetadataAt !== null;
+ const { contentType, metadata: probeMetadata }: UrlProbeResult =
+ precrawledArchiveAssetId
+ ? { contentType: ASSET_TYPES.TEXT_HTML, metadata: Promise.resolve(null) }
+ : await getContentTypeAndMetadata(url, jobId, job.abortSignal, runProxy, {
+ skipMetadataExtraction: reuseStoredProbeMetadata,
+ });
job.abortSignal.throwIfAborted();
// Link bookmarks get transformed into asset bookmarks if they point to a supported asset instead of a webpage
@@ -2404,6 +2686,31 @@ async function runCrawler(
runProxy,
);
} else {
+ // Chain the early metadata write onto the (still running) probe
+ // extraction so a title/thumbnail lands as soon as it's ready, while the
+ // browser crawl proceeds in parallel. crawlAndParseUrl awaits this same
+ // promise before its own metadata write, so within this attempt the
+ // early write always precedes the final merged one. Across attempts the
+ // ordering can't be relied on (a failed attempt's write may fire after a
+ // retry finished) — that case is guarded by the write itself being
+ // fill-only (see writeProbeMetadata). On retry runs where the probe
+ // metadata was already stored, reload it from the bookmark row instead.
+ // Never rejects.
+ const probeMetadataPromise = reuseStoredProbeMetadata
+ ? loadStoredProbeMetadata(bookmarkId, jobId)
+ : probeMetadata.then(async (metadata) => {
+ if (metadata) {
+ const { error } = await tryCatch(
+ writeProbeMetadata(bookmarkId, metadata, jobId),
+ );
+ if (error) {
+ logger.warn(
+ `[Crawler][${jobId}] Failed to write early probe metadata: ${error}`,
+ );
+ }
+ }
+ return metadata;
+ });
const archivalLogic = await crawlAndParseUrl(
url,
userId,
@@ -2420,6 +2727,7 @@ async function runCrawler(
numRetriesLeft,
job.abortSignal,
runProxy,
+ probeMetadataPromise,
);
// Propagate priority to child jobs
diff --git a/apps/workers/workers/utils/metadataResolver.test.ts b/apps/workers/workers/utils/metadataResolver.test.ts
new file mode 100644
index 00000000..74cdf7f1
--- /dev/null
+++ b/apps/workers/workers/utils/metadataResolver.test.ts
@@ -0,0 +1,140 @@
+import { describe, expect, it } from "vitest";
+
+import { isLikelyChallengePage, resolveMetadata } from "./metadataResolver";
+
+function meta(fields: Record<string, string | null | undefined>) {
+ return {
+ title: null,
+ description: null,
+ image: null,
+ logo: null,
+ author: null,
+ publisher: null,
+ datePublished: null,
+ dateModified: null,
+ ...fields,
+ };
+}
+
+describe("resolveMetadata", () => {
+ it("returns render metadata untouched when there is no probe metadata", () => {
+ const render = meta({ title: "Render Title" });
+ expect(resolveMetadata(render, null, false)).toBe(render);
+ expect(resolveMetadata(render, null, true)).toBe(render);
+ });
+
+ it("prefers render values and fills gaps from the probe on a healthy render", () => {
+ const resolved = resolveMetadata(
+ meta({ title: "Render Title", image: null }),
+ meta({ title: "Probe Title", image: "https://example.com/probe.png" }),
+ false,
+ );
+ expect(resolved.title).toBe("Render Title");
+ expect(resolved.image).toBe("https://example.com/probe.png");
+ });
+
+ it("prefers probe values and fills gaps from the render on a blocked render", () => {
+ const resolved = resolveMetadata(
+ meta({ title: "Just a moment...", author: "Render Author" }),
+ meta({ title: "Probe Title", author: null }),
+ true,
+ );
+ expect(resolved.title).toBe("Probe Title");
+ expect(resolved.author).toBe("Render Author");
+ });
+
+ it("treats empty strings as missing", () => {
+ const resolved = resolveMetadata(
+ meta({ title: "", description: "" }),
+ meta({ title: "Probe Title" }),
+ false,
+ );
+ expect(resolved.title).toBe("Probe Title");
+ expect(resolved.description).toBeNull();
+ });
+
+ it("preserves extra fields from the render metadata", () => {
+ const resolved = resolveMetadata(
+ meta({ title: "Render Title", lang: "en" }),
+ meta({ title: "Probe Title" }),
+ false,
+ );
+ expect(resolved.lang).toBe("en");
+ });
+
+ it("does not mutate its inputs", () => {
+ const render = meta({ title: null });
+ const probe = meta({ title: "Probe Title" });
+ resolveMetadata(render, probe, false);
+ expect(render.title).toBeNull();
+ expect(probe.title).toBe("Probe Title");
+ });
+});
+
+describe("isLikelyChallengePage", () => {
+ it("detects known challenge-page titles regardless of case and whitespace", () => {
+ expect(isLikelyChallengePage({ title: "Just a moment..." })).toBe(true);
+ expect(isLikelyChallengePage({ title: " JUST A MOMENT... " })).toBe(true);
+ expect(
+ isLikelyChallengePage({ title: "Attention Required! | Cloudflare" }),
+ ).toBe(true);
+ expect(isLikelyChallengePage({ title: "Access Denied" })).toBe(true);
+ expect(isLikelyChallengePage({ title: "Robot Check" })).toBe(true);
+ });
+
+ it("does not match challenge titles as substrings of real titles", () => {
+ expect(
+ isLikelyChallengePage({
+ title: "Access denied: a deep dive into HTTP 403 semantics",
+ }),
+ ).toBe(false);
+ });
+
+ it("detects challenge markers in small page bodies", () => {
+ expect(
+ isLikelyChallengePage({
+ title: "example.com",
+ htmlContent: `<html><body><script>window._cf_chl_opt = {};</script></body></html>`,
+ }),
+ ).toBe(true);
+ expect(
+ isLikelyChallengePage({
+ htmlContent: `<html><body><div id="px-captcha"></div></body></html>`,
+ }),
+ ).toBe(true);
+ expect(
+ isLikelyChallengePage({
+ htmlContent: `<html><body><iframe src="https://geo.captcha-delivery.com/captcha/"></iframe></body></html>`,
+ }),
+ ).toBe(true);
+ expect(
+ isLikelyChallengePage({
+ htmlContent: `<html><body><p>Checking your browser before accessing example.com</p></body></html>`,
+ }),
+ ).toBe(true);
+ });
+
+ it("ignores body markers in large documents (e.g. articles quoting them)", () => {
+ const article =
+ `<html><body><p>Cloudflare shows "checking your browser before accessing" on its interstitial.</p>` +
+ `<p>${"lorem ipsum ".repeat(10_000)}</p></body></html>`;
+ expect(isLikelyChallengePage({ htmlContent: article })).toBe(false);
+ });
+
+ it("does not flag normal pages that merely mention bot-protection vendors", () => {
+ expect(
+ isLikelyChallengePage({
+ title: "How Cloudflare and DataDome detect bots",
+ htmlContent:
+ "<html><body><p>An overview of Cloudflare, DataDome, PerimeterX and Akamai bot protection.</p></body></html>",
+ }),
+ ).toBe(false);
+ });
+
+ it("returns false for missing inputs", () => {
+ expect(isLikelyChallengePage({})).toBe(false);
+ expect(isLikelyChallengePage({ title: null, htmlContent: null })).toBe(
+ false,
+ );
+ });
+});
diff --git a/apps/workers/workers/utils/metadataResolver.ts b/apps/workers/workers/utils/metadataResolver.ts
new file mode 100644
index 00000000..da1f1e7f
--- /dev/null
+++ b/apps/workers/workers/utils/metadataResolver.ts
@@ -0,0 +1,116 @@
+import type { ParseSubprocessOutput } from "./parseHtmlSubprocessIpc";
+
+/**
+ * Exact (lowercased, trimmed) page titles used by common bot-protection
+ * interstitials and block pages. Matched exactly, not as substrings, so a
+ * legitimate article about bot walls doesn't trip the heuristic.
+ */
+const CHALLENGE_PAGE_TITLES = new Set([
+ // Cloudflare
+ "just a moment...",
+ "attention required! | cloudflare",
+ "one moment, please...",
+ // Akamai / generic block pages
+ "access denied",
+ // PerimeterX
+ "access to this page has been denied",
+ "access to this page has been denied.",
+ // Amazon / Bloomberg
+ "robot check",
+ "are you a robot?",
+ // Distil Networks / Imperva
+ "pardon our interruption",
+ // Generic captcha interstitials
+ "human verification",
+ "verify you are human",
+ "verifying you are human",
+ "bot verification",
+ // Vercel
+ "vercel security checkpoint",
+]);
+
+/**
+ * High-confidence markers that only appear in the markup of an actual
+ * challenge/block page (captcha iframes, challenge bootstrap globals,
+ * interstitial copy) — not in the vendor's loader snippets that protected
+ * sites embed on every page.
+ */
+const CHALLENGE_PAGE_BODY_MARKERS = [
+ // Cloudflare interstitial
+ "cf_chl_opt",
+ "cf-browser-verification",
+ "cf-challenge-running",
+ "checking your browser before accessing",
+ "enable javascript and cookies to continue",
+ "verifying you are human",
+ // PerimeterX captcha block page
+ "px-captcha",
+ // DataDome captcha host (only referenced from its block page)
+ "captcha-delivery.com",
+ // Akamai / Apache-style forbidden pages
+ "you don't have permission to access",
+];
+
+/**
+ * Challenge and block pages are tiny; real articles are not. Body markers are
+ * only searched in small documents so an article merely *quoting* interstitial
+ * copy (e.g. a post about bot protection) isn't misclassified.
+ */
+const CHALLENGE_PAGE_MAX_HTML_LENGTH = 100_000;
+
+/**
+ * Conservative detection of bot-challenge/block pages that are served with an
+ * HTTP 200 and would otherwise look like a successful crawl. Prefers false
+ * negatives over false positives: exact title matches, plus high-confidence
+ * body markers gated on the document being challenge-page sized.
+ */
+export function isLikelyChallengePage({
+ title,
+ htmlContent,
+}: {
+ title?: string | null;
+ htmlContent?: string | null;
+}): boolean {
+ if (title && CHALLENGE_PAGE_TITLES.has(title.trim().toLowerCase())) {
+ return true;
+ }
+ if (!htmlContent || htmlContent.length > CHALLENGE_PAGE_MAX_HTML_LENGTH) {
+ return false;
+ }
+ const lowered = htmlContent.toLowerCase();
+ return CHALLENGE_PAGE_BODY_MARKERS.some((marker) => lowered.includes(marker));
+}
+
+/**
+ * Merges the browser render's metadata with the preflight probe's, per field.
+ * The render normally wins (it sees the post-JS DOM), with the probe filling
+ * gaps. But when the render was blocked (retryable status code on the last
+ * retry attempt, or a challenge page served with a 200), its metadata is
+ * likely from a challenge or error page, so the probe's values take
+ * precedence instead and the render only fills gaps.
+ */
+export function resolveMetadata(
+ renderMeta: ParseSubprocessOutput["metadata"],
+ probeMeta: ParseSubprocessOutput["metadata"] | null,
+ renderBlocked: boolean,
+): ParseSubprocessOutput["metadata"] {
+ if (!probeMeta) {
+ return renderMeta;
+ }
+ const pick = (
+ render: string | null | undefined,
+ probe: string | null | undefined,
+ ): string | null =>
+ (renderBlocked ? probe || render : render || probe) || null;
+ return {
+ ...renderMeta,
+ title: pick(renderMeta.title, probeMeta.title),
+ description: pick(renderMeta.description, probeMeta.description),
+ image: pick(renderMeta.image, probeMeta.image),
+ logo: pick(renderMeta.logo, probeMeta.logo),
+ author: pick(renderMeta.author, probeMeta.author),
+ publisher: pick(renderMeta.publisher, probeMeta.publisher),
+ datePublished: pick(renderMeta.datePublished, probeMeta.datePublished),
+ dateModified: pick(renderMeta.dateModified, probeMeta.dateModified),
+ };
+}
diff --git a/apps/workers/workers/utils/parseHtmlSubprocessIpc.ts b/apps/workers/workers/utils/parseHtmlSubprocessIpc.ts
index cc87b457..304830f2 100644
--- a/apps/workers/workers/utils/parseHtmlSubprocessIpc.ts
+++ b/apps/workers/workers/utils/parseHtmlSubprocessIpc.ts
@@ -4,6 +4,9 @@ export const parseSubprocessInputSchema = z.object({
htmlContent: z.string(),
url: z.string(),
jobId: z.string(),
+ // When true, only run metadata extraction and skip the (expensive)
+ // readable-content extraction. `readableContent` will be null.
+ metadataOnly: z.boolean().optional(),
});
export const parseSubprocessMetadataSchema = z.looseObject({
diff --git a/docs/docs/03-configuration/01-environment-variables.md b/docs/docs/03-configuration/01-environment-variables.md
index 5d143594..39db763b 100644
--- a/docs/docs/03-configuration/01-environment-variables.md
+++ b/docs/docs/03-configuration/01-environment-variables.md
@@ -143,6 +143,7 @@ Either `OPENAI_API_KEY` or `OLLAMA_BASE_URL` need to be set for automatic taggin
| CRAWLER_FULL_PAGE_ARCHIVE | No | false | Whether to store a full local copy of the page or not. Disabled by default, as it can lead to much higher disk usage. If disabled, only the readable text of the page is archived. |
| CRAWLER_JOB_TIMEOUT_SEC | No | 60 | How long to wait for the crawler job to finish before timing out. If you have a slow internet connection or a low powered device, you might want to bump this up a bit |
| CRAWLER_NAVIGATE_TIMEOUT_SEC | No | 30 | How long to spend navigating to the page (along with its redirects). Increase this if you have a slow internet connection |
+| CRAWLER_PREFLIGHT_USER_AGENT | No | Not set | The `User-Agent` header to send on the crawler's preflight request (the plain HTTP request that determines a link's content type and extracts early preview metadata before the full crawl). If not set, no `User-Agent` header is sent. |
| CRAWLER_PARSE_TIMEOUT_SEC | No | 60 | How long to wait for the HTML parsing subprocess (metadata extraction + readability) to finish before timing out. |
| CRAWLER_PARSER_MEM_LIMIT_MB | No | 512 | Maximum heap size in MB for the HTML parsing subprocess. If the parser OOMs on large pages, increase this value. The subprocess is isolated so an OOM won't crash the main worker. |
| CRAWLER_VIDEO_DOWNLOAD | No | false | Whether to download videos from the page or not (using yt-dlp) |
diff --git a/packages/db/drizzle/0086_bookmark_links_probe_metadata_at.sql b/packages/db/drizzle/0086_bookmark_links_probe_metadata_at.sql
new file mode 100644
index 00000000..e6258737
--- /dev/null
+++ b/packages/db/drizzle/0086_bookmark_links_probe_metadata_at.sql
@@ -0,0 +1 @@
+ALTER TABLE `bookmarkLinks` ADD `probeMetadataAt` integer;
\ No newline at end of file
diff --git a/packages/db/drizzle/meta/0086_snapshot.json b/packages/db/drizzle/meta/0086_snapshot.json
new file mode 100644
index 00000000..290f46ff
--- /dev/null
+++ b/packages/db/drizzle/meta/0086_snapshot.json
@@ -0,0 +1,3363 @@
+{
+ "version": "6",
+ "dialect": "sqlite",
+ "id": "3e2d6b09-5a4e-4ddb-b44b-70ded16e2ae0",
+ "prevId": "240f4be9-65f7-4176-9376-fd2daa9b443c",
+ "tables": {
+ "account": {
+ "name": "account",
+ "columns": {
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "type": {
+ "name": "type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "provider": {
+ "name": "provider",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "providerAccountId": {
+ "name": "providerAccountId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "refresh_token": {
+ "name": "refresh_token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "access_token": {
+ "name": "access_token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "token_type": {
+ "name": "token_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "scope": {
+ "name": "scope",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "id_token": {
+ "name": "id_token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "session_state": {
+ "name": "session_state",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "account_userId_user_id_fk": {
+ "name": "account_userId_user_id_fk",
+ "tableFrom": "account",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "account_provider_providerAccountId_pk": {
+ "columns": [
+ "provider",
+ "providerAccountId"
+ ],
+ "name": "account_provider_providerAccountId_pk"
+ }
+ },
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "apiKey": {
+ "name": "apiKey",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "lastUsedAt": {
+ "name": "lastUsedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "keyId": {
+ "name": "keyId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "keyHash": {
+ "name": "keyHash",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "scopes": {
+ "name": "scopes",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "apiKey_keyId_unique": {
+ "name": "apiKey_keyId_unique",
+ "columns": [
+ "keyId"
+ ],
+ "isUnique": true
+ },
+ "apiKey_name_userId_unique": {
+ "name": "apiKey_name_userId_unique",
+ "columns": [
+ "name",
+ "userId"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {
+ "apiKey_userId_user_id_fk": {
+ "name": "apiKey_userId_user_id_fk",
+ "tableFrom": "apiKey",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "assets": {
+ "name": "assets",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "assetType": {
+ "name": "assetType",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "size": {
+ "name": "size",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "contentType": {
+ "name": "contentType",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "fileName": {
+ "name": "fileName",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "bookmarkId": {
+ "name": "bookmarkId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "assets_bookmarkId_idx": {
+ "name": "assets_bookmarkId_idx",
+ "columns": [
+ "bookmarkId"
+ ],
+ "isUnique": false
+ },
+ "assets_assetType_idx": {
+ "name": "assets_assetType_idx",
+ "columns": [
+ "assetType"
+ ],
+ "isUnique": false
+ },
+ "assets_userId_idx": {
+ "name": "assets_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "assets_bookmarkId_bookmarks_id_fk": {
+ "name": "assets_bookmarkId_bookmarks_id_fk",
+ "tableFrom": "assets",
+ "tableTo": "bookmarks",
+ "columnsFrom": [
+ "bookmarkId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "assets_userId_user_id_fk": {
+ "name": "assets_userId_user_id_fk",
+ "tableFrom": "assets",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "backups": {
+ "name": "backups",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "assetId": {
+ "name": "assetId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "size": {
+ "name": "size",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "bookmarkCount": {
+ "name": "bookmarkCount",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "errorMessage": {
+ "name": "errorMessage",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "backups_userId_idx": {
+ "name": "backups_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ },
+ "backups_createdAt_idx": {
+ "name": "backups_createdAt_idx",
+ "columns": [
+ "createdAt"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "backups_userId_user_id_fk": {
+ "name": "backups_userId_user_id_fk",
+ "tableFrom": "backups",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "backups_assetId_assets_id_fk": {
+ "name": "backups_assetId_assets_id_fk",
+ "tableFrom": "backups",
+ "tableTo": "assets",
+ "columnsFrom": [
+ "assetId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "bookmarkAssets": {
+ "name": "bookmarkAssets",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "assetType": {
+ "name": "assetType",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "assetId": {
+ "name": "assetId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "content": {
+ "name": "content",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "metadata": {
+ "name": "metadata",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "fileName": {
+ "name": "fileName",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "sourceUrl": {
+ "name": "sourceUrl",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "bookmarkAssets_id_bookmarks_id_fk": {
+ "name": "bookmarkAssets_id_bookmarks_id_fk",
+ "tableFrom": "bookmarkAssets",
+ "tableTo": "bookmarks",
+ "columnsFrom": [
+ "id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "bookmarkLinks": {
+ "name": "bookmarkLinks",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "url": {
+ "name": "url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "title": {
+ "name": "title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "author": {
+ "name": "author",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "publisher": {
+ "name": "publisher",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "datePublished": {
+ "name": "datePublished",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "dateModified": {
+ "name": "dateModified",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "imageUrl": {
+ "name": "imageUrl",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "favicon": {
+ "name": "favicon",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "htmlContent": {
+ "name": "htmlContent",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "contentAssetId": {
+ "name": "contentAssetId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "crawledAt": {
+ "name": "crawledAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "crawlStatus": {
+ "name": "crawlStatus",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "crawlStatusCode": {
+ "name": "crawlStatusCode",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": 200
+ },
+ "probeMetadataAt": {
+ "name": "probeMetadataAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "bookmarkLinks_url_idx": {
+ "name": "bookmarkLinks_url_idx",
+ "columns": [
+ "url"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "bookmarkLinks_id_bookmarks_id_fk": {
+ "name": "bookmarkLinks_id_bookmarks_id_fk",
+ "tableFrom": "bookmarkLinks",
+ "tableTo": "bookmarks",
+ "columnsFrom": [
+ "id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "bookmarkLists": {
+ "name": "bookmarkLists",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "icon": {
+ "name": "icon",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "type": {
+ "name": "type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "query": {
+ "name": "query",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "parentId": {
+ "name": "parentId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "rssToken": {
+ "name": "rssToken",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "public": {
+ "name": "public",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ }
+ },
+ "indexes": {
+ "bookmarkLists_userId_idx": {
+ "name": "bookmarkLists_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ },
+ "bookmarkLists_userId_id_idx": {
+ "name": "bookmarkLists_userId_id_idx",
+ "columns": [
+ "userId",
+ "id"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {
+ "bookmarkLists_userId_user_id_fk": {
+ "name": "bookmarkLists_userId_user_id_fk",
+ "tableFrom": "bookmarkLists",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "bookmarkLists_parentId_bookmarkLists_id_fk": {
+ "name": "bookmarkLists_parentId_bookmarkLists_id_fk",
+ "tableFrom": "bookmarkLists",
+ "tableTo": "bookmarkLists",
+ "columnsFrom": [
+ "parentId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "bookmarkTags": {
+ "name": "bookmarkTags",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "normalizedName": {
+ "name": "normalizedName",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "generated": {
+ "as": "(lower(replace(replace(replace(\"name\", ' ', ''), '-', ''), '_', '')))",
+ "type": "virtual"
+ }
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "bookmarkTags_name_idx": {
+ "name": "bookmarkTags_name_idx",
+ "columns": [
+ "name"
+ ],
+ "isUnique": false
+ },
+ "bookmarkTags_userId_idx": {
+ "name": "bookmarkTags_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ },
+ "bookmarkTags_normalizedName_idx": {
+ "name": "bookmarkTags_normalizedName_idx",
+ "columns": [
+ "normalizedName"
+ ],
+ "isUnique": false
+ },
+ "bookmarkTags_userId_name_unique": {
+ "name": "bookmarkTags_userId_name_unique",
+ "columns": [
+ "userId",
+ "name"
+ ],
+ "isUnique": true
+ },
+ "bookmarkTags_userId_id_idx": {
+ "name": "bookmarkTags_userId_id_idx",
+ "columns": [
+ "userId",
+ "id"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {
+ "bookmarkTags_userId_user_id_fk": {
+ "name": "bookmarkTags_userId_user_id_fk",
+ "tableFrom": "bookmarkTags",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "bookmarkTexts": {
+ "name": "bookmarkTexts",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "text": {
+ "name": "text",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "sourceUrl": {
+ "name": "sourceUrl",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "bookmarkTexts_id_bookmarks_id_fk": {
+ "name": "bookmarkTexts_id_bookmarks_id_fk",
+ "tableFrom": "bookmarkTexts",
+ "tableTo": "bookmarks",
+ "columnsFrom": [
+ "id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "bookmarks": {
+ "name": "bookmarks",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "modifiedAt": {
+ "name": "modifiedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "title": {
+ "name": "title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "archived": {
+ "name": "archived",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ },
+ "favourited": {
+ "name": "favourited",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "taggingStatus": {
+ "name": "taggingStatus",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "summarizationStatus": {
+ "name": "summarizationStatus",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "embeddingStatus": {
+ "name": "embeddingStatus",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "summary": {
+ "name": "summary",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "note": {
+ "name": "note",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "type": {
+ "name": "type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "source": {
+ "name": "source",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "bookmarks_userId_idx": {
+ "name": "bookmarks_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ },
+ "bookmarks_createdAt_idx": {
+ "name": "bookmarks_createdAt_idx",
+ "columns": [
+ "createdAt"
+ ],
+ "isUnique": false
+ },
+ "bookmarks_userId_createdAt_id_idx": {
+ "name": "bookmarks_userId_createdAt_id_idx",
+ "columns": [
+ "userId",
+ "createdAt",
+ "id"
+ ],
+ "isUnique": false
+ },
+ "bookmarks_userId_archived_createdAt_id_idx": {
+ "name": "bookmarks_userId_archived_createdAt_id_idx",
+ "columns": [
+ "userId",
+ "archived",
+ "createdAt",
+ "id"
+ ],
+ "isUnique": false
+ },
+ "bookmarks_userId_favourited_createdAt_id_idx": {
+ "name": "bookmarks_userId_favourited_createdAt_id_idx",
+ "columns": [
+ "userId",
+ "favourited",
+ "createdAt",
+ "id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "bookmarks_userId_user_id_fk": {
+ "name": "bookmarks_userId_user_id_fk",
+ "tableFrom": "bookmarks",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "bookmarksInLists": {
+ "name": "bookmarksInLists",
+ "columns": {
+ "bookmarkId": {
+ "name": "bookmarkId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "listId": {
+ "name": "listId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "addedAt": {
+ "name": "addedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "listMembershipId": {
+ "name": "listMembershipId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "bookmarksInLists_bookmarkId_idx": {
+ "name": "bookmarksInLists_bookmarkId_idx",
+ "columns": [
+ "bookmarkId"
+ ],
+ "isUnique": false
+ },
+ "bookmarksInLists_listId_idx": {
+ "name": "bookmarksInLists_listId_idx",
+ "columns": [
+ "listId"
+ ],
+ "isUnique": false
+ },
+ "bookmarksInLists_listId_bookmarkId_idx": {
+ "name": "bookmarksInLists_listId_bookmarkId_idx",
+ "columns": [
+ "listId",
+ "bookmarkId"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "bookmarksInLists_bookmarkId_bookmarks_id_fk": {
+ "name": "bookmarksInLists_bookmarkId_bookmarks_id_fk",
+ "tableFrom": "bookmarksInLists",
+ "tableTo": "bookmarks",
+ "columnsFrom": [
+ "bookmarkId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "bookmarksInLists_listId_bookmarkLists_id_fk": {
+ "name": "bookmarksInLists_listId_bookmarkLists_id_fk",
+ "tableFrom": "bookmarksInLists",
+ "tableTo": "bookmarkLists",
+ "columnsFrom": [
+ "listId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "bookmarksInLists_listMembershipId_listCollaborators_id_fk": {
+ "name": "bookmarksInLists_listMembershipId_listCollaborators_id_fk",
+ "tableFrom": "bookmarksInLists",
+ "tableTo": "listCollaborators",
+ "columnsFrom": [
+ "listMembershipId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "bookmarksInLists_bookmarkId_listId_pk": {
+ "columns": [
+ "bookmarkId",
+ "listId"
+ ],
+ "name": "bookmarksInLists_bookmarkId_listId_pk"
+ }
+ },
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "config": {
+ "name": "config",
+ "columns": {
+ "key": {
+ "name": "key",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "value": {
+ "name": "value",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "customPrompts": {
+ "name": "customPrompts",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "text": {
+ "name": "text",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "enabled": {
+ "name": "enabled",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "appliesTo": {
+ "name": "appliesTo",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "customPrompts_userId_idx": {
+ "name": "customPrompts_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "customPrompts_userId_user_id_fk": {
+ "name": "customPrompts_userId_user_id_fk",
+ "tableFrom": "customPrompts",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "highlights": {
+ "name": "highlights",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "bookmarkId": {
+ "name": "bookmarkId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "startOffset": {
+ "name": "startOffset",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "endOffset": {
+ "name": "endOffset",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "color": {
+ "name": "color",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'yellow'"
+ },
+ "text": {
+ "name": "text",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "note": {
+ "name": "note",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "highlights_bookmarkId_idx": {
+ "name": "highlights_bookmarkId_idx",
+ "columns": [
+ "bookmarkId"
+ ],
+ "isUnique": false
+ },
+ "highlights_userId_idx": {
+ "name": "highlights_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "highlights_bookmarkId_bookmarks_id_fk": {
+ "name": "highlights_bookmarkId_bookmarks_id_fk",
+ "tableFrom": "highlights",
+ "tableTo": "bookmarks",
+ "columnsFrom": [
+ "bookmarkId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "highlights_userId_user_id_fk": {
+ "name": "highlights_userId_user_id_fk",
+ "tableFrom": "highlights",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "importSessionBookmarks": {
+ "name": "importSessionBookmarks",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "importSessionId": {
+ "name": "importSessionId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "bookmarkId": {
+ "name": "bookmarkId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "importSessionBookmarks_sessionId_idx": {
+ "name": "importSessionBookmarks_sessionId_idx",
+ "columns": [
+ "importSessionId"
+ ],
+ "isUnique": false
+ },
+ "importSessionBookmarks_bookmarkId_idx": {
+ "name": "importSessionBookmarks_bookmarkId_idx",
+ "columns": [
+ "bookmarkId"
+ ],
+ "isUnique": false
+ },
+ "importSessionBookmarks_importSessionId_bookmarkId_unique": {
+ "name": "importSessionBookmarks_importSessionId_bookmarkId_unique",
+ "columns": [
+ "importSessionId",
+ "bookmarkId"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {
+ "importSessionBookmarks_importSessionId_importSessions_id_fk": {
+ "name": "importSessionBookmarks_importSessionId_importSessions_id_fk",
+ "tableFrom": "importSessionBookmarks",
+ "tableTo": "importSessions",
+ "columnsFrom": [
+ "importSessionId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "importSessionBookmarks_bookmarkId_bookmarks_id_fk": {
+ "name": "importSessionBookmarks_bookmarkId_bookmarks_id_fk",
+ "tableFrom": "importSessionBookmarks",
+ "tableTo": "bookmarks",
+ "columnsFrom": [
+ "bookmarkId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "importSessions": {
+ "name": "importSessions",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "message": {
+ "name": "message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "rootListId": {
+ "name": "rootListId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'staging'"
+ },
+ "lastProcessedAt": {
+ "name": "lastProcessedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "modifiedAt": {
+ "name": "modifiedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "importSessions_userId_idx": {
+ "name": "importSessions_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ },
+ "importSessions_status_idx": {
+ "name": "importSessions_status_idx",
+ "columns": [
+ "status"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "importSessions_userId_user_id_fk": {
+ "name": "importSessions_userId_user_id_fk",
+ "tableFrom": "importSessions",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "importSessions_rootListId_bookmarkLists_id_fk": {
+ "name": "importSessions_rootListId_bookmarkLists_id_fk",
+ "tableFrom": "importSessions",
+ "tableTo": "bookmarkLists",
+ "columnsFrom": [
+ "rootListId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "importStagingBookmarks": {
+ "name": "importStagingBookmarks",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "importSessionId": {
+ "name": "importSessionId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "type": {
+ "name": "type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "url": {
+ "name": "url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "title": {
+ "name": "title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "content": {
+ "name": "content",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "note": {
+ "name": "note",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "tags": {
+ "name": "tags",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "listIds": {
+ "name": "listIds",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "sourceAddedAt": {
+ "name": "sourceAddedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "archived": {
+ "name": "archived",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "processingStartedAt": {
+ "name": "processingStartedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "result": {
+ "name": "result",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "resultReason": {
+ "name": "resultReason",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "resultBookmarkId": {
+ "name": "resultBookmarkId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "completedAt": {
+ "name": "completedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "importStaging_session_status_idx": {
+ "name": "importStaging_session_status_idx",
+ "columns": [
+ "importSessionId",
+ "status"
+ ],
+ "isUnique": false
+ },
+ "importStaging_completedAt_idx": {
+ "name": "importStaging_completedAt_idx",
+ "columns": [
+ "completedAt"
+ ],
+ "isUnique": false
+ },
+ "importStaging_status_idx": {
+ "name": "importStaging_status_idx",
+ "columns": [
+ "status"
+ ],
+ "isUnique": false
+ },
+ "importStaging_status_processingStartedAt_idx": {
+ "name": "importStaging_status_processingStartedAt_idx",
+ "columns": [
+ "status",
+ "processingStartedAt"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "importStagingBookmarks_importSessionId_importSessions_id_fk": {
+ "name": "importStagingBookmarks_importSessionId_importSessions_id_fk",
+ "tableFrom": "importStagingBookmarks",
+ "tableTo": "importSessions",
+ "columnsFrom": [
+ "importSessionId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "importStagingBookmarks_resultBookmarkId_bookmarks_id_fk": {
+ "name": "importStagingBookmarks_resultBookmarkId_bookmarks_id_fk",
+ "tableFrom": "importStagingBookmarks",
+ "tableTo": "bookmarks",
+ "columnsFrom": [
+ "resultBookmarkId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "invites": {
+ "name": "invites",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "token": {
+ "name": "token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "usedAt": {
+ "name": "usedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "invitedBy": {
+ "name": "invitedBy",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "invites_token_unique": {
+ "name": "invites_token_unique",
+ "columns": [
+ "token"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {
+ "invites_invitedBy_user_id_fk": {
+ "name": "invites_invitedBy_user_id_fk",
+ "tableFrom": "invites",
+ "tableTo": "user",
+ "columnsFrom": [
+ "invitedBy"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "listCollaborators": {
+ "name": "listCollaborators",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "listId": {
+ "name": "listId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "role": {
+ "name": "role",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "addedBy": {
+ "name": "addedBy",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "listCollaborators_listId_idx": {
+ "name": "listCollaborators_listId_idx",
+ "columns": [
+ "listId"
+ ],
+ "isUnique": false
+ },
+ "listCollaborators_userId_idx": {
+ "name": "listCollaborators_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ },
+ "listCollaborators_listId_userId_unique": {
+ "name": "listCollaborators_listId_userId_unique",
+ "columns": [
+ "listId",
+ "userId"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {
+ "listCollaborators_listId_bookmarkLists_id_fk": {
+ "name": "listCollaborators_listId_bookmarkLists_id_fk",
+ "tableFrom": "listCollaborators",
+ "tableTo": "bookmarkLists",
+ "columnsFrom": [
+ "listId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "listCollaborators_userId_user_id_fk": {
+ "name": "listCollaborators_userId_user_id_fk",
+ "tableFrom": "listCollaborators",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "listCollaborators_addedBy_user_id_fk": {
+ "name": "listCollaborators_addedBy_user_id_fk",
+ "tableFrom": "listCollaborators",
+ "tableTo": "user",
+ "columnsFrom": [
+ "addedBy"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "listInvitations": {
+ "name": "listInvitations",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "listId": {
+ "name": "listId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "role": {
+ "name": "role",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "invitedAt": {
+ "name": "invitedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "invitedEmail": {
+ "name": "invitedEmail",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "invitedBy": {
+ "name": "invitedBy",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "listInvitations_listId_idx": {
+ "name": "listInvitations_listId_idx",
+ "columns": [
+ "listId"
+ ],
+ "isUnique": false
+ },
+ "listInvitations_userId_idx": {
+ "name": "listInvitations_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ },
+ "listInvitations_status_idx": {
+ "name": "listInvitations_status_idx",
+ "columns": [
+ "status"
+ ],
+ "isUnique": false
+ },
+ "listInvitations_listId_userId_unique": {
+ "name": "listInvitations_listId_userId_unique",
+ "columns": [
+ "listId",
+ "userId"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {
+ "listInvitations_listId_bookmarkLists_id_fk": {
+ "name": "listInvitations_listId_bookmarkLists_id_fk",
+ "tableFrom": "listInvitations",
+ "tableTo": "bookmarkLists",
+ "columnsFrom": [
+ "listId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "listInvitations_userId_user_id_fk": {
+ "name": "listInvitations_userId_user_id_fk",
+ "tableFrom": "listInvitations",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "listInvitations_invitedBy_user_id_fk": {
+ "name": "listInvitations_invitedBy_user_id_fk",
+ "tableFrom": "listInvitations",
+ "tableTo": "user",
+ "columnsFrom": [
+ "invitedBy"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "passwordResetToken": {
+ "name": "passwordResetToken",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "token": {
+ "name": "token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "expires": {
+ "name": "expires",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "passwordResetToken_token_unique": {
+ "name": "passwordResetToken_token_unique",
+ "columns": [
+ "token"
+ ],
+ "isUnique": true
+ },
+ "passwordResetTokens_userId_idx": {
+ "name": "passwordResetTokens_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "passwordResetToken_userId_user_id_fk": {
+ "name": "passwordResetToken_userId_user_id_fk",
+ "tableFrom": "passwordResetToken",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "rssFeedImports": {
+ "name": "rssFeedImports",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "entryId": {
+ "name": "entryId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "rssFeedId": {
+ "name": "rssFeedId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "bookmarkId": {
+ "name": "bookmarkId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "rssFeedImports_feedIdIdx_idx": {
+ "name": "rssFeedImports_feedIdIdx_idx",
+ "columns": [
+ "rssFeedId"
+ ],
+ "isUnique": false
+ },
+ "rssFeedImports_entryIdIdx_idx": {
+ "name": "rssFeedImports_entryIdIdx_idx",
+ "columns": [
+ "entryId"
+ ],
+ "isUnique": false
+ },
+ "rssFeedImports_rssFeedId_bookmarkId_idx": {
+ "name": "rssFeedImports_rssFeedId_bookmarkId_idx",
+ "columns": [
+ "rssFeedId",
+ "bookmarkId"
+ ],
+ "isUnique": false
+ },
+ "rssFeedImports_rssFeedId_entryId_unique": {
+ "name": "rssFeedImports_rssFeedId_entryId_unique",
+ "columns": [
+ "rssFeedId",
+ "entryId"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {
+ "rssFeedImports_rssFeedId_rssFeeds_id_fk": {
+ "name": "rssFeedImports_rssFeedId_rssFeeds_id_fk",
+ "tableFrom": "rssFeedImports",
+ "tableTo": "rssFeeds",
+ "columnsFrom": [
+ "rssFeedId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "rssFeedImports_bookmarkId_bookmarks_id_fk": {
+ "name": "rssFeedImports_bookmarkId_bookmarks_id_fk",
+ "tableFrom": "rssFeedImports",
+ "tableTo": "bookmarks",
+ "columnsFrom": [
+ "bookmarkId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "rssFeeds": {
+ "name": "rssFeeds",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "url": {
+ "name": "url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "enabled": {
+ "name": "enabled",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": true
+ },
+ "importTags": {
+ "name": "importTags",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "lastFetchedAt": {
+ "name": "lastFetchedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "lastSuccessfulFetchAt": {
+ "name": "lastSuccessfulFetchAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "lastFetchedStatus": {
+ "name": "lastFetchedStatus",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "rssFeeds_userId_idx": {
+ "name": "rssFeeds_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "rssFeeds_userId_user_id_fk": {
+ "name": "rssFeeds_userId_user_id_fk",
+ "tableFrom": "rssFeeds",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "ruleEngineActions": {
+ "name": "ruleEngineActions",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "ruleId": {
+ "name": "ruleId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "action": {
+ "name": "action",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "listId": {
+ "name": "listId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "tagId": {
+ "name": "tagId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "ruleEngineActions_userId_idx": {
+ "name": "ruleEngineActions_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ },
+ "ruleEngineActions_ruleId_idx": {
+ "name": "ruleEngineActions_ruleId_idx",
+ "columns": [
+ "ruleId"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "ruleEngineActions_userId_user_id_fk": {
+ "name": "ruleEngineActions_userId_user_id_fk",
+ "tableFrom": "ruleEngineActions",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "ruleEngineActions_ruleId_ruleEngineRules_id_fk": {
+ "name": "ruleEngineActions_ruleId_ruleEngineRules_id_fk",
+ "tableFrom": "ruleEngineActions",
+ "tableTo": "ruleEngineRules",
+ "columnsFrom": [
+ "ruleId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "ruleEngineActions_userId_tagId_fk": {
+ "name": "ruleEngineActions_userId_tagId_fk",
+ "tableFrom": "ruleEngineActions",
+ "tableTo": "bookmarkTags",
+ "columnsFrom": [
+ "userId",
+ "tagId"
+ ],
+ "columnsTo": [
+ "userId",
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "ruleEngineActions_userId_listId_fk": {
+ "name": "ruleEngineActions_userId_listId_fk",
+ "tableFrom": "ruleEngineActions",
+ "tableTo": "bookmarkLists",
+ "columnsFrom": [
+ "userId",
+ "listId"
+ ],
+ "columnsTo": [
+ "userId",
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "ruleEngineRules": {
+ "name": "ruleEngineRules",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "enabled": {
+ "name": "enabled",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": true
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "event": {
+ "name": "event",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "condition": {
+ "name": "condition",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "tagId": {
+ "name": "tagId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "ruleEngine_userId_idx": {
+ "name": "ruleEngine_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "ruleEngineRules_userId_user_id_fk": {
+ "name": "ruleEngineRules_userId_user_id_fk",
+ "tableFrom": "ruleEngineRules",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "ruleEngineRules_userId_tagId_fk": {
+ "name": "ruleEngineRules_userId_tagId_fk",
+ "tableFrom": "ruleEngineRules",
+ "tableTo": "bookmarkTags",
+ "columnsFrom": [
+ "userId",
+ "tagId"
+ ],
+ "columnsTo": [
+ "userId",
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "session": {
+ "name": "session",
+ "columns": {
+ "sessionToken": {
+ "name": "sessionToken",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "expires": {
+ "name": "expires",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "session_userId_user_id_fk": {
+ "name": "session_userId_user_id_fk",
+ "tableFrom": "session",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "subscriptions": {
+ "name": "subscriptions",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "stripeCustomerId": {
+ "name": "stripeCustomerId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "stripeSubscriptionId": {
+ "name": "stripeSubscriptionId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "tier": {
+ "name": "tier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'free'"
+ },
+ "priceId": {
+ "name": "priceId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "cancelAtPeriodEnd": {
+ "name": "cancelAtPeriodEnd",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": false
+ },
+ "startDate": {
+ "name": "startDate",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "endDate": {
+ "name": "endDate",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "modifiedAt": {
+ "name": "modifiedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "subscriptions_userId_unique": {
+ "name": "subscriptions_userId_unique",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": true
+ },
+ "subscriptions_userId_idx": {
+ "name": "subscriptions_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ },
+ "subscriptions_stripeCustomerId_idx": {
+ "name": "subscriptions_stripeCustomerId_idx",
+ "columns": [
+ "stripeCustomerId"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "subscriptions_userId_user_id_fk": {
+ "name": "subscriptions_userId_user_id_fk",
+ "tableFrom": "subscriptions",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "tagsOnBookmarks": {
+ "name": "tagsOnBookmarks",
+ "columns": {
+ "bookmarkId": {
+ "name": "bookmarkId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "tagId": {
+ "name": "tagId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "attachedAt": {
+ "name": "attachedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "attachedBy": {
+ "name": "attachedBy",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "tagsOnBookmarks_tagId_idx": {
+ "name": "tagsOnBookmarks_tagId_idx",
+ "columns": [
+ "tagId"
+ ],
+ "isUnique": false
+ },
+ "tagsOnBookmarks_bookmarkId_idx": {
+ "name": "tagsOnBookmarks_bookmarkId_idx",
+ "columns": [
+ "bookmarkId"
+ ],
+ "isUnique": false
+ },
+ "tagsOnBookmarks_tagId_bookmarkId_idx": {
+ "name": "tagsOnBookmarks_tagId_bookmarkId_idx",
+ "columns": [
+ "tagId",
+ "bookmarkId"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "tagsOnBookmarks_bookmarkId_bookmarks_id_fk": {
+ "name": "tagsOnBookmarks_bookmarkId_bookmarks_id_fk",
+ "tableFrom": "tagsOnBookmarks",
+ "tableTo": "bookmarks",
+ "columnsFrom": [
+ "bookmarkId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "tagsOnBookmarks_tagId_bookmarkTags_id_fk": {
+ "name": "tagsOnBookmarks_tagId_bookmarkTags_id_fk",
+ "tableFrom": "tagsOnBookmarks",
+ "tableTo": "bookmarkTags",
+ "columnsFrom": [
+ "tagId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "tagsOnBookmarks_bookmarkId_tagId_pk": {
+ "columns": [
+ "bookmarkId",
+ "tagId"
+ ],
+ "name": "tagsOnBookmarks_bookmarkId_tagId_pk"
+ }
+ },
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "userReadingProgress": {
+ "name": "userReadingProgress",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "bookmarkId": {
+ "name": "bookmarkId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "readingProgressOffset": {
+ "name": "readingProgressOffset",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "readingProgressAnchor": {
+ "name": "readingProgressAnchor",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "readingProgressPercent": {
+ "name": "readingProgressPercent",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "modifiedAt": {
+ "name": "modifiedAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "userReadingProgress_bookmarkId_idx": {
+ "name": "userReadingProgress_bookmarkId_idx",
+ "columns": [
+ "bookmarkId"
+ ],
+ "isUnique": false
+ },
+ "userReadingProgress_userId_idx": {
+ "name": "userReadingProgress_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ },
+ "userReadingProgress_bookmarkId_userId_unique": {
+ "name": "userReadingProgress_bookmarkId_userId_unique",
+ "columns": [
+ "bookmarkId",
+ "userId"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {
+ "userReadingProgress_bookmarkId_bookmarks_id_fk": {
+ "name": "userReadingProgress_bookmarkId_bookmarks_id_fk",
+ "tableFrom": "userReadingProgress",
+ "tableTo": "bookmarks",
+ "columnsFrom": [
+ "bookmarkId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "userReadingProgress_userId_user_id_fk": {
+ "name": "userReadingProgress_userId_user_id_fk",
+ "tableFrom": "userReadingProgress",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "user": {
+ "name": "user",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "emailVerified": {
+ "name": "emailVerified",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "image": {
+ "name": "image",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "salt": {
+ "name": "salt",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "''"
+ },
+ "role": {
+ "name": "role",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": "'user'"
+ },
+ "bookmarkQuota": {
+ "name": "bookmarkQuota",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "storageQuota": {
+ "name": "storageQuota",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "browserCrawlingEnabled": {
+ "name": "browserCrawlingEnabled",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "bookmarkClickAction": {
+ "name": "bookmarkClickAction",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'open_original_link'"
+ },
+ "archiveDisplayBehaviour": {
+ "name": "archiveDisplayBehaviour",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'show'"
+ },
+ "timezone": {
+ "name": "timezone",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": "'UTC'"
+ },
+ "backupsEnabled": {
+ "name": "backupsEnabled",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ },
+ "backupsFrequency": {
+ "name": "backupsFrequency",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'weekly'"
+ },
+ "backupsRetentionDays": {
+ "name": "backupsRetentionDays",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 30
+ },
+ "readerFontSize": {
+ "name": "readerFontSize",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "readerLineHeight": {
+ "name": "readerLineHeight",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "readerFontFamily": {
+ "name": "readerFontFamily",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "autoTaggingEnabled": {
+ "name": "autoTaggingEnabled",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "autoSummarizationEnabled": {
+ "name": "autoSummarizationEnabled",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "tagStyle": {
+ "name": "tagStyle",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": "'titlecase-spaces'"
+ },
+ "curatedTagIds": {
+ "name": "curatedTagIds",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "inferredTagLang": {
+ "name": "inferredTagLang",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "user_email_unique": {
+ "name": "user_email_unique",
+ "columns": [
+ "email"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "verificationToken": {
+ "name": "verificationToken",
+ "columns": {
+ "identifier": {
+ "name": "identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "token": {
+ "name": "token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "expires": {
+ "name": "expires",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {
+ "verificationToken_identifier_token_pk": {
+ "columns": [
+ "identifier",
+ "token"
+ ],
+ "name": "verificationToken_identifier_token_pk"
+ }
+ },
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "webhooks": {
+ "name": "webhooks",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "createdAt": {
+ "name": "createdAt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "url": {
+ "name": "url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "userId": {
+ "name": "userId",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "events": {
+ "name": "events",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "token": {
+ "name": "token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "webhooks_userId_idx": {
+ "name": "webhooks_userId_idx",
+ "columns": [
+ "userId"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "webhooks_userId_user_id_fk": {
+ "name": "webhooks_userId_user_id_fk",
+ "tableFrom": "webhooks",
+ "tableTo": "user",
+ "columnsFrom": [
+ "userId"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ }
+ },
+ "views": {},
+ "enums": {},
+ "_meta": {
+ "schemas": {},
+ "tables": {},
+ "columns": {}
+ },
+ "internal": {
+ "indexes": {}
+ }
+}
\ No newline at end of file
diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json
index 7e6ef631..39e2dab1 100644
--- a/packages/db/drizzle/meta/_journal.json
+++ b/packages/db/drizzle/meta/_journal.json
@@ -603,6 +603,13 @@
"when": 1780154803212,
"tag": "0085_add_embedding_status",
"breakpoints": true
+ },
+ {
+ "idx": 86,
+ "version": "6",
+ "when": 1783262454462,
+ "tag": "0086_bookmark_links_probe_metadata_at",
+ "breakpoints": true
}
]
}
\ No newline at end of file
diff --git a/packages/db/schema.ts b/packages/db/schema.ts
index 20a5a78c..ad5199c8 100644
--- a/packages/db/schema.ts
+++ b/packages/db/schema.ts
@@ -275,6 +275,9 @@ export const bookmarkLinks = sqliteTable(
enum: ["pending", "failure", "success"],
}).default("pending"),
crawlStatusCode: integer("crawlStatusCode").default(200),
+ // When the pre-crawl probe last extracted and stored this link's metadata.
+ // Lets crawl retries skip re-fetching it.
+ probeMetadataAt: integer("probeMetadataAt", { mode: "timestamp" }),
},
(bl) => [index("bookmarkLinks_url_idx").on(bl.url)],
);
diff --git a/packages/shared-server/src/eventLogTypes.ts b/packages/shared-server/src/eventLogTypes.ts
index 1fd10a7a..c9df5847 100644
--- a/packages/shared-server/src/eventLogTypes.ts
+++ b/packages/shared-server/src/eventLogTypes.ts
@@ -6,6 +6,17 @@ type EventLogInternal =
"crawler.domain"?: string;
"crawler.status_code"?: number | null;
"crawler.proxy"?: string;
+ // Outcome of the preflight probe's metadata extraction.
+ "crawler.probe.metadata"?:
+ | "extracted"
+ | "blocked_status"
+ | "challenge_page"
+ | "reused_stored"
+ | "failed";
+ // Whether the rendered page was considered blocked (retryable status
+ // code or a detected challenge page), flipping metadata precedence to
+ // the preflight probe.
+ "crawler.render_blocked"?: boolean;
}
| {
["event.name"]: "inferenceWorker.run";
diff --git a/packages/shared/config.ts b/packages/shared/config.ts
index 2a2b96ba..12ce939c 100644
--- a/packages/shared/config.ts
+++ b/packages/shared/config.ts
@@ -132,6 +132,7 @@ const allEnv = z.object({
CRAWLER_IP_VALIDATION_DNS_RESOLVER_TIMEOUT_SEC: z.coerce.number().default(1),
CRAWLER_DOMAIN_RATE_LIMIT_WINDOW_MS: z.coerce.number().min(1).optional(),
CRAWLER_DOMAIN_RATE_LIMIT_MAX_REQUESTS: z.coerce.number().min(1).optional(),
+ CRAWLER_PREFLIGHT_USER_AGENT: z.string().optional(),
LOG_LEVEL: z.string().default("debug"),
NO_COLOR: stringBool("false"),
DEMO_MODE: stringBool("false"),
@@ -375,6 +376,7 @@ const serverConfigSchema = allEnv.transform((val, ctx) => {
maxRequests: val.CRAWLER_DOMAIN_RATE_LIMIT_MAX_REQUESTS,
}
: null,
+ preflightUserAgent: val.CRAWLER_PREFLIGHT_USER_AGENT,
},
ocr: {
langs: val.OCR_LANGS,