Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

feat(web): improve broken links management (#3105)

Mohamed Bassem93b478314dcf
diff --git a/apps/web/app/settings/broken-links/page.tsx b/apps/web/app/settings/broken-links/page.tsx
index 6f8f8eae..dc95384c 100644
--- a/apps/web/app/settings/broken-links/page.tsx
+++ b/apps/web/app/settings/broken-links/page.tsx
@@ -1,12 +1,26 @@
"use client";
+import { useMemo, useState } from "react";
+import Link from "next/link";
import {
- SettingsPage,
- SettingsSection,
-} from "@/components/settings/SettingsPage";
+ MaintenanceEmpty,
+ MaintenanceError,
+ MaintenanceSearch,
+ SelectionCheckbox,
+} from "@/components/settings/Maintenance";
+import { SettingsPage } from "@/components/settings/SettingsPage";
import { ActionButton } from "@/components/ui/action-button";
+import ActionConfirmingDialog from "@/components/ui/action-confirming-dialog";
+import { Button } from "@/components/ui/button";
import FormattedDate from "@/components/ui/formatted-date";
-import { FullPageSpinner } from "@/components/ui/full-page-spinner";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
+import { Skeleton } from "@/components/ui/skeleton";
import { toast } from "@/components/ui/sonner";
import {
Table,
@@ -16,125 +30,531 @@ import {
TableHeader,
TableRow,
} from "@/components/ui/table";
-import { useQuery, useQueryClient } from "@tanstack/react-query";
-import { RefreshCw, Trash2 } from "lucide-react";
-import { useTranslation } from "react-i18next";
-
import {
- useDeleteBookmark,
- useRecrawlBookmark,
-} from "@karakeep/shared-react/hooks/bookmarks";
+ getBrokenLinkIssue,
+ getLinkHostname,
+ stripUrlProtocol,
+} from "@/lib/broken-links";
+import { useTranslation } from "@/lib/i18n/client";
+import { cn } from "@/lib/utils";
+import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
+import {
+ ArrowUpRight,
+ Check,
+ CircleCheck,
+ CircleHelp,
+ Link2Off,
+ RefreshCw,
+ ShieldAlert,
+ Trash2,
+} from "lucide-react";
+
import { useTRPC } from "@karakeep/shared-react/trpc";
+import { limitConcurrency } from "@karakeep/shared/concurrency";
+import { MAX_CONCURRENT_BULK_ACTIONS } from "@/lib/hooks/useBookmarkBulkActions";
+
+const categories = ["all", "blocked", "missing", "other"] as const;
+const categoryIcons = {
+ all: Link2Off,
+ blocked: ShieldAlert,
+ missing: Link2Off,
+ other: CircleHelp,
+};
+type Category = (typeof categories)[number];
+const issueClass = {
+ blocked: "bg-amber-500/10 text-amber-700 dark:text-amber-400",
+ missing: "bg-rose-500/10 text-rose-700 dark:text-rose-400",
+ server: "bg-orange-500/10 text-orange-700 dark:text-orange-400",
+ failed: "bg-muted text-muted-foreground",
+ http: "bg-muted text-muted-foreground",
+};
+
+function IssueBadge({ statusCode }: { statusCode: number | null }) {
+ const { t } = useTranslation();
+ const issue = getBrokenLinkIssue(statusCode);
+ return (
+ <span
+ title={t(`settings.broken_links.issue_${issue}_hint`)}
+ className={cn(
+ "inline-flex items-center gap-1.5 whitespace-nowrap rounded-md px-2 py-1 text-xs font-medium",
+ issueClass[issue],
+ )}
+ >
+ {statusCode !== null && (statusCode < 200 || statusCode >= 300) && (
+ <span className="font-mono">{statusCode}</span>
+ )}
+ {t(`settings.broken_links.issue_${issue}`)}
+ </span>
+ );
+}
export default function BrokenLinksPage() {
const api = useTRPC();
const { t } = useTranslation();
-
const queryClient = useQueryClient();
- const { data, isPending } = useQuery(
- api.bookmarks.getBrokenLinks.queryOptions(),
+ const [search, setSearch] = useState("");
+ const [category, setCategory] = useState<Category>("all");
+ const [sort, setSort] = useState<"recent" | "oldest" | "domain">("recent");
+ const [selected, setSelected] = useState<Set<string>>(new Set());
+ const [busy, setBusy] = useState(false);
+ const [deleting, setDeleting] = useState<string[]>([]);
+ const { data, isPending, isError, refetch } = useQuery(
+ api.bookmarks.getBrokenLinks.queryOptions(undefined, {
+ refetchInterval: (query) =>
+ query.state.data?.bookmarks.some((bookmark) => bookmark.isCrawling)
+ ? 10000
+ : false,
+ }),
+ );
+ const { mutateAsync: recrawl } = useMutation(
+ api.bookmarks.recrawlBookmark.mutationOptions(),
+ );
+ const { mutateAsync: remove } = useMutation(
+ api.bookmarks.deleteBookmark.mutationOptions(),
);
+ const bookmarks = data?.bookmarks ?? [];
+ const counts = useMemo(() => {
+ const result = { all: bookmarks.length, blocked: 0, missing: 0, other: 0 };
+ for (const bookmark of bookmarks) {
+ const issue = getBrokenLinkIssue(bookmark.statusCode);
+ result[issue === "blocked" || issue === "missing" ? issue : "other"]++;
+ }
+ return result;
+ }, [bookmarks]);
+ const searchQuery = search.trim().toLowerCase();
+ const filtered = useMemo(
+ () =>
+ bookmarks
+ .filter((bookmark) => {
+ const issue = getBrokenLinkIssue(bookmark.statusCode);
+ return (
+ (category === "all" ||
+ (category === "other"
+ ? issue !== "blocked" && issue !== "missing"
+ : issue === category)) &&
+ (!searchQuery ||
+ `${bookmark.title ?? ""} ${bookmark.url}`
+ .toLowerCase()
+ .includes(searchQuery))
+ );
+ })
+ .sort((a, b) =>
+ sort === "domain"
+ ? getLinkHostname(a.url).localeCompare(getLinkHostname(b.url))
+ : sort === "oldest"
+ ? (a.crawledAt?.getTime() ?? 0) - (b.crawledAt?.getTime() ?? 0)
+ : (b.crawledAt?.getTime() ?? 0) - (a.crawledAt?.getTime() ?? 0),
+ ),
+ [bookmarks, category, searchQuery, sort],
+ );
+ const selection = filtered.filter((bookmark) => selected.has(bookmark.id));
+ const hasFilters = !!searchQuery || category !== "all";
+ const clearFilters = () => {
+ setSearch("");
+ setCategory("all");
+ setSelected(new Set());
+ };
- const { mutate: deleteBookmark, isPending: isDeleting } = useDeleteBookmark({
- onSuccess: () => {
- toast({
- description: t("toasts.bookmarks.deleted"),
- });
- queryClient.invalidateQueries(api.bookmarks.getBrokenLinks.pathFilter());
- },
- onError: () => {
+ async function runAction(ids: string[], action: "retry" | "delete") {
+ setBusy(true);
+ try {
+ const results = await Promise.allSettled(
+ limitConcurrency(
+ ids.map(
+ (id) => () =>
+ action === "retry"
+ ? recrawl({ bookmarkId: id })
+ : remove({ bookmarkId: id }),
+ ),
+ MAX_CONCURRENT_BULK_ACTIONS,
+ ),
+ );
+ await Promise.all([
+ queryClient.invalidateQueries(api.bookmarks.pathFilter()),
+ ...(action === "delete"
+ ? [
+ queryClient.invalidateQueries(api.assets.list.pathFilter()),
+ queryClient.invalidateQueries(api.lists.stats.pathFilter()),
+ ]
+ : []),
+ ]);
+ const failedIds = ids.filter(
+ (_, index) => results[index].status === "rejected",
+ );
+ const failed = failedIds.length;
+ setSelected(new Set(failedIds));
+ if (action === "delete") setDeleting(failedIds);
toast({
- description: t("common.something_went_wrong"),
- variant: "destructive",
+ description: failed
+ ? t("settings.broken_links.action_failed", {
+ count: failed,
+ total: ids.length,
+ })
+ : t(
+ action === "retry"
+ ? "settings.broken_links.queued_count"
+ : "settings.broken_links.deleted_count",
+ { count: ids.length - failed },
+ ),
+ variant: failed ? "destructive" : "default",
});
- },
- });
-
- const { mutate: recrawlBookmark, isPending: isRecrawling } =
- useRecrawlBookmark({
- onSuccess: () => {
- toast({
- description: t("toasts.bookmarks.refetch"),
- });
- queryClient.invalidateQueries(
- api.bookmarks.getBrokenLinks.pathFilter(),
- );
- },
- onError: () => {
- toast({
- description: t("common.something_went_wrong"),
- variant: "destructive",
- });
- },
- });
+ } finally {
+ setBusy(false);
+ }
+ }
return (
- <SettingsPage title={t("settings.broken_links.broken_links")}>
- <SettingsSection>
- {isPending && <FullPageSpinner />}
- {!isPending && data && data.bookmarks.length == 0 && (
- <p className="rounded-md bg-muted p-3 text-center text-sm text-muted-foreground">
- No broken links found
+ <SettingsPage
+ title={t("settings.broken_links.broken_links")}
+ description={t("settings.broken_links.description")}
+ >
+ <div className="grid grid-cols-2 gap-3 lg:grid-cols-4">
+ {categories.map((value) => {
+ const Icon = categoryIcons[value];
+ return (
+ <button
+ key={value}
+ aria-pressed={category === value}
+ onClick={() => {
+ setCategory(value);
+ setSelected(new Set());
+ }}
+ className={cn(
+ "rounded-xl border bg-card p-4 text-left transition-colors hover:border-muted-foreground/40 focus-visible:outline focus-visible:outline-2 focus-visible:outline-ring sm:p-5",
+ category === value &&
+ "border-blue-200 bg-blue-50 dark:border-blue-800 dark:bg-blue-950/40",
+ )}
+ >
+ <div className="flex items-center justify-between gap-2 text-sm text-muted-foreground">
+ <span>{t(`settings.broken_links.category_${value}`)}</span>
+ <Icon
+ className={cn(
+ "size-4",
+ value === "blocked" && "text-amber-500",
+ value === "missing" && "text-rose-500",
+ )}
+ />
+ </div>
+ {isPending ? (
+ <Skeleton className="mt-3 h-8 w-12" />
+ ) : (
+ <p className="mt-2 text-3xl font-semibold tabular-nums tracking-tight">
+ {data ? counts[value] : "—"}
+ </p>
+ )}
+ <p className="mt-1 text-xs text-muted-foreground">
+ {t(`settings.broken_links.category_${value}_hint`)}
+ </p>
+ </button>
+ );
+ })}
+ </div>
+ <div className="flex items-start gap-3 rounded-lg border border-amber-500/20 bg-amber-500/5 px-4 py-3 text-sm">
+ <ShieldAlert className="mt-0.5 size-4 shrink-0 text-amber-600 dark:text-amber-400" />
+ <p className="text-muted-foreground">
+ {t("settings.broken_links.blocked_hint")}
+ </p>
+ </div>
+ <section className="overflow-hidden rounded-xl border bg-card">
+ <div className="flex flex-col gap-3 border-b p-4 sm:flex-row">
+ <MaintenanceSearch
+ aria-label={t("settings.broken_links.search")}
+ placeholder={t("settings.broken_links.search")}
+ value={search}
+ onChange={(event) => {
+ setSearch(event.target.value);
+ setSelected(new Set());
+ }}
+ />
+ <Select
+ value={sort}
+ onValueChange={(value) => setSort(value as typeof sort)}
+ >
+ <SelectTrigger
+ className="w-full sm:w-48"
+ aria-label={t("settings.maintenance.sort")}
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {(["recent", "oldest", "domain"] as const).map((value) => (
+ <SelectItem key={value} value={value}>
+ {t(`settings.broken_links.sort_${value}`)}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+ <div className="flex min-h-14 flex-wrap items-center justify-between gap-2 border-b bg-muted/30 px-4 py-2 text-sm">
+ <p className="text-muted-foreground" aria-live="polite">
+ {selection.length
+ ? t("settings.maintenance.selected", { count: selection.length })
+ : t("settings.broken_links.result_count", {
+ count: filtered.length,
+ })}
</p>
- )}
- {!isPending && data && data.bookmarks.length > 0 && (
+ <div className="flex flex-wrap gap-2">
+ {selection.length > 0 ? (
+ <>
+ <Button
+ variant="ghost"
+ size="sm"
+ disabled={busy}
+ onClick={() => setSelected(new Set())}
+ >
+ {t("settings.maintenance.clear_selection")}
+ </Button>
+ <ActionButton
+ variant="outline"
+ size="sm"
+ loading={busy && deleting.length === 0}
+ disabled={busy || !selection.length}
+ onClick={() =>
+ void runAction(
+ selection.map((bookmark) => bookmark.id),
+ "retry",
+ )
+ }
+ >
+ <RefreshCw className="mr-2 size-3.5" />
+ {t("settings.broken_links.retry_selected")}
+ </ActionButton>
+ <Button
+ variant="ghostDestructive"
+ size="sm"
+ disabled={busy}
+ onClick={() =>
+ setDeleting(selection.map((bookmark) => bookmark.id))
+ }
+ >
+ <Trash2 className="mr-2 size-3.5" />
+ {t("settings.maintenance.delete_selected")}
+ </Button>
+ </>
+ ) : (
+ hasFilters && (
+ <Button variant="ghost" size="sm" onClick={clearFilters}>
+ {t("settings.maintenance.clear_filters")}
+ </Button>
+ )
+ )}
+ </div>
+ </div>
+ {isError ? (
+ <div className="p-4">
+ <MaintenanceError onRetry={() => void refetch()} />
+ </div>
+ ) : isPending ? (
+ <div className="space-y-4 p-5">
+ {[0, 1, 2, 3].map((i) => (
+ <Skeleton key={i} className="h-14 w-full" />
+ ))}
+ </div>
+ ) : !filtered.length ? (
+ <MaintenanceEmpty
+ icon={
+ hasFilters ? (
+ <Link2Off className="size-6" />
+ ) : (
+ <CircleCheck className="size-6" />
+ )
+ }
+ title={t(
+ hasFilters
+ ? "settings.maintenance.no_matches"
+ : "settings.broken_links.empty",
+ )}
+ description={t(
+ hasFilters
+ ? "settings.maintenance.adjust_filters"
+ : "settings.broken_links.empty_description",
+ )}
+ onReset={hasFilters ? clearFilters : undefined}
+ />
+ ) : (
<Table>
<TableHeader>
- <TableRow>
- <TableHead>{t("common.url")}</TableHead>
- <TableHead>{t("common.created_at")}</TableHead>
- <TableHead>
+ <TableRow className="hover:bg-transparent">
+ <TableHead className="w-10 pr-0">
+ <SelectionCheckbox
+ aria-label={t("settings.broken_links.select_visible")}
+ checked={
+ selection.length === filtered.length &&
+ filtered.length > 0
+ }
+ indeterminate={
+ selection.length > 0 && selection.length < filtered.length
+ }
+ disabled={busy}
+ onChange={(event) =>
+ setSelected(
+ event.target.checked
+ ? new Set(filtered.map((bookmark) => bookmark.id))
+ : new Set(),
+ )
+ }
+ />
+ </TableHead>
+ <TableHead>{t("settings.broken_links.bookmark")}</TableHead>
+ <TableHead>{t("settings.broken_links.issue")}</TableHead>
+ <TableHead className="whitespace-nowrap">
{t("settings.broken_links.last_crawled_at")}
</TableHead>
<TableHead>
- {t("settings.broken_links.crawling_status")}
+ <span className="sr-only">{t("common.actions")}</span>
</TableHead>
- <TableHead>{t("common.action")}</TableHead>
</TableRow>
</TableHeader>
<TableBody>
- {data.bookmarks.map((b) => (
- <TableRow key={b.id}>
- <TableCell>{b.url}</TableCell>
- <TableCell>
- <FormattedDate date={b.createdAt} />
- </TableCell>
- <TableCell>
- <FormattedDate date={b.crawledAt} />
- </TableCell>
- <TableCell>
- {b.isCrawlingFailure ? (
- <span className="text-red-500">Failed</span>
- ) : (
- b.statusCode
- )}
- </TableCell>
- <TableCell className="flex gap-2">
- <ActionButton
- variant="secondary"
- loading={isRecrawling}
- onClick={() => recrawlBookmark({ bookmarkId: b.id })}
- className="flex items-center gap-2"
- >
- <RefreshCw className="size-4" />
- {t("actions.recrawl")}
- </ActionButton>
- <ActionButton
- variant="ghostDestructive"
- onClick={() => deleteBookmark({ bookmarkId: b.id })}
- loading={isDeleting}
- className="flex items-center gap-2"
- >
- <Trash2 className="size-4" />
- </ActionButton>
- </TableCell>
- </TableRow>
- ))}
- <TableRow></TableRow>
+ {filtered.map((bookmark) => {
+ const isCrawling = bookmark.isCrawling;
+ return (
+ <TableRow
+ key={bookmark.id}
+ className="data-[state=selected]:bg-blue-50 dark:data-[state=selected]:bg-blue-950/40"
+ data-state={
+ selected.has(bookmark.id) ? "selected" : undefined
+ }
+ >
+ <TableCell className="pr-0">
+ <SelectionCheckbox
+ aria-label={t("settings.maintenance.select_item", {
+ name: bookmark.title || bookmark.url,
+ })}
+ checked={selected.has(bookmark.id)}
+ disabled={busy}
+ onChange={(event) =>
+ setSelected((current) => {
+ const next = new Set(current);
+ if (event.target.checked) next.add(bookmark.id);
+ else next.delete(bookmark.id);
+ return next;
+ })
+ }
+ />
+ </TableCell>
+ <TableCell className="w-full max-w-0 px-2 sm:px-4">
+ <Link
+ href={`/dashboard/preview/${bookmark.id}`}
+ prefetch={false}
+ className="block truncate font-medium hover:underline"
+ title={bookmark.title || bookmark.url}
+ >
+ {bookmark.title || getLinkHostname(bookmark.url)}
+ </Link>
+ <a
+ href={bookmark.url}
+ target="_blank"
+ rel="noopener noreferrer"
+ className="mt-1 flex items-center gap-1 text-xs text-muted-foreground hover:text-foreground"
+ title={bookmark.url}
+ >
+ <span className="truncate">
+ {stripUrlProtocol(bookmark.url)}
+ </span>
+ <ArrowUpRight className="size-3 shrink-0" />
+ </a>
+ </TableCell>
+ <TableCell className="whitespace-nowrap">
+ <IssueBadge statusCode={bookmark.statusCode} />
+ </TableCell>
+ <TableCell className="whitespace-nowrap text-xs text-muted-foreground">
+ {bookmark.crawledAt ? (
+ <FormattedDate
+ date={bookmark.crawledAt}
+ formatStr="PP"
+ />
+ ) : (
+ t("settings.broken_links.never_crawled")
+ )}
+ </TableCell>
+ <TableCell className="px-2 sm:px-4">
+ <div className="flex items-center gap-1">
+ <ActionButton
+ variant="ghost"
+ size="sm"
+ loading={false}
+ disabled={busy || isCrawling}
+ onClick={() => void runAction([bookmark.id], "retry")}
+ title={t(
+ isCrawling
+ ? "settings.broken_links.queued"
+ : "actions.recrawl",
+ )}
+ aria-label={t(
+ isCrawling
+ ? "settings.broken_links.queued"
+ : "actions.recrawl",
+ )}
+ className={cn(
+ "gap-2 px-2 sm:px-3",
+ isCrawling && "text-teal-600 dark:text-teal-400",
+ )}
+ >
+ {isCrawling ? (
+ <Check className="size-4" />
+ ) : (
+ <RefreshCw className="size-4" />
+ )}
+ <span className="hidden sm:inline">
+ {t(
+ isCrawling
+ ? "settings.broken_links.queued"
+ : "actions.recrawl",
+ )}
+ </span>
+ </ActionButton>
+ <Button
+ variant="ghost"
+ size="icon"
+ disabled={busy}
+ onClick={() => setDeleting([bookmark.id])}
+ aria-label={t("actions.delete")}
+ title={t("actions.delete")}
+ className="text-muted-foreground hover:bg-destructive/10 hover:text-destructive"
+ >
+ <Trash2 className="size-4" />
+ </Button>
+ </div>
+ </TableCell>
+ </TableRow>
+ );
+ })}
</TableBody>
</Table>
)}
- </SettingsSection>
+ {!!filtered.length && (
+ <div className="border-t px-5 py-3 text-xs text-muted-foreground">
+ {t("settings.maintenance.showing", {
+ shown: filtered.length,
+ total: bookmarks.length,
+ })}
+ </div>
+ )}
+ </section>
+ <ActionConfirmingDialog
+ open={deleting.length > 0}
+ setOpen={(open) => {
+ if (!open && !busy) setDeleting([]);
+ }}
+ title={t("settings.broken_links.delete_title", {
+ count: deleting.length,
+ })}
+ description={
+ <p className="text-sm text-muted-foreground">
+ {t("settings.broken_links.delete_description")}
+ </p>
+ }
+ actionButton={() => (
+ <ActionButton
+ variant="destructive"
+ loading={busy}
+ onClick={() => void runAction(deleting, "delete")}
+ >
+ {t("actions.delete")}
+ </ActionButton>
+ )}
+ />
</SettingsPage>
);
}
diff --git a/apps/web/components/settings/Maintenance.tsx b/apps/web/components/settings/Maintenance.tsx
new file mode 100644
index 00000000..a7488a94
--- /dev/null
+++ b/apps/web/components/settings/Maintenance.tsx
@@ -0,0 +1,94 @@
+"use client";
+
+import type { ComponentProps, ReactNode } from "react";
+import { useEffect, useRef } from "react";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { useTranslation } from "@/lib/i18n/client";
+import { cn } from "@/lib/utils";
+import { Search } from "lucide-react";
+
+export function SelectionCheckbox({
+ indeterminate,
+ className,
+ ...props
+}: ComponentProps<"input"> & { indeterminate?: boolean }) {
+ const ref = useRef<HTMLInputElement>(null);
+ useEffect(() => {
+ if (ref.current) ref.current.indeterminate = !!indeterminate;
+ }, [indeterminate]);
+ return (
+ <input
+ {...props}
+ ref={ref}
+ type="checkbox"
+ className={cn(
+ "size-4 shrink-0 cursor-pointer accent-primary disabled:cursor-default disabled:opacity-30",
+ className,
+ )}
+ />
+ );
+}
+
+export function MaintenanceSearch(props: ComponentProps<typeof Input>) {
+ return (
+ <div className="min-w-0 flex-1">
+ <Input
+ {...props}
+ type="search"
+ startIcon={
+ <Search aria-hidden="true" className="size-4 text-muted-foreground" />
+ }
+ />
+ </div>
+ );
+}
+
+export function MaintenanceEmpty({
+ icon,
+ title,
+ description,
+ onReset,
+}: {
+ icon: ReactNode;
+ title: string;
+ description?: string;
+ onReset?: () => void;
+}) {
+ const { t } = useTranslation();
+ return (
+ <div className="flex flex-col items-center gap-3 px-6 py-14 text-center">
+ <div className="rounded-full bg-muted p-4 text-muted-foreground">
+ {icon}
+ </div>
+ <div>
+ <p className="font-medium">{title}</p>
+ {description && (
+ <p className="mt-1 max-w-md text-sm text-muted-foreground">
+ {description}
+ </p>
+ )}
+ </div>
+ {onReset && (
+ <Button variant="outline" size="sm" onClick={onReset}>
+ {t("settings.maintenance.clear_filters")}
+ </Button>
+ )}
+ </div>
+ );
+}
+
+export function MaintenanceError({ onRetry }: { onRetry: () => void }) {
+ const { t } = useTranslation();
+ return (
+ <div
+ role="alert"
+ className="flex flex-wrap items-center justify-between gap-3 rounded-lg border border-destructive/20 bg-destructive/5 p-4 text-sm"
+ >
+ <p>{t("common.something_went_wrong")}</p>
+ <Button variant="outline" size="sm" onClick={onRetry}>
+ {t("settings.maintenance.try_again")}
+ </Button>
+ </div>
+ );
+}
diff --git a/apps/web/lib/broken-links.ts b/apps/web/lib/broken-links.ts
new file mode 100644
index 00000000..8636f02b
--- /dev/null
+++ b/apps/web/lib/broken-links.ts
@@ -0,0 +1,31 @@
+export type BrokenLinkIssue =
+ | "blocked"
+ | "missing"
+ | "server"
+ | "http"
+ | "failed";
+
+const BLOCKED_STATUS_CODES = new Set([401, 403, 429]);
+const MISSING_STATUS_CODES = new Set([404, 410]);
+
+export function getBrokenLinkIssue(statusCode: number | null): BrokenLinkIssue {
+ if (statusCode !== null) {
+ if (BLOCKED_STATUS_CODES.has(statusCode)) return "blocked";
+ if (MISSING_STATUS_CODES.has(statusCode)) return "missing";
+ if (statusCode >= 500) return "server";
+ if (statusCode < 200 || statusCode >= 300) return "http";
+ }
+ return "failed";
+}
+
+export function getLinkHostname(url: string) {
+ try {
+ return new URL(url).hostname.replace(/^www\./, "");
+ } catch {
+ return url;
+ }
+}
+
+export function stripUrlProtocol(url: string) {
+ return url.replace(/^https?:\/\//, "");
+}
diff --git a/apps/web/lib/hooks/useBookmarkBulkActions.ts b/apps/web/lib/hooks/useBookmarkBulkActions.ts
index 4a877557..7b8c715f 100644
--- a/apps/web/lib/hooks/useBookmarkBulkActions.ts
+++ b/apps/web/lib/hooks/useBookmarkBulkActions.ts
@@ -13,7 +13,7 @@ import {
} from "@karakeep/shared-react/hooks/bookmarks";
import { useRemoveBookmarkFromList } from "@karakeep/shared-react/hooks/lists";
-const MAX_CONCURRENT_BULK_ACTIONS = 50;
+export const MAX_CONCURRENT_BULK_ACTIONS = 50;
export interface UpdateBookmarkProps {
favourited?: boolean;
diff --git a/apps/web/lib/i18n/locales/en/translation.json b/apps/web/lib/i18n/locales/en/translation.json
index b9eb41b1..cfd0ded5 100644
--- a/apps/web/lib/i18n/locales/en/translation.json
+++ b/apps/web/lib/i18n/locales/en/translation.json
@@ -428,7 +428,49 @@
"broken_links": "Broken Links",
"last_crawled_at": "Last Crawled At",
"crawling_status": "Crawling Status",
- "crawling_failed": "Crawling Failed"
+ "crawling_failed": "Crawling Failed",
+ "description": "Review links that could not be fetched, retry them or remove old bookmarks.",
+ "category_all": "Needs attention",
+ "category_blocked": "Access blocked",
+ "category_missing": "Not found",
+ "category_other": "Other errors",
+ "category_all_hint": "Links to review",
+ "category_blocked_hint": "Restricted or rate limited",
+ "category_missing_hint": "Page missing or removed",
+ "category_other_hint": "Server or crawl failures",
+ "blocked_hint": "A blocked request doesn't always mean a broken link. Some sites restrict automated access. Open the original page to check before deleting.",
+ "search": "Search by title or URL…",
+ "sort_recent": "Recently checked",
+ "sort_oldest": "Oldest check first",
+ "sort_domain": "Website A–Z",
+ "result_count_one": "{{count}} link to review",
+ "result_count_other": "{{count}} links to review",
+ "retry_selected": "Retry selected",
+ "empty": "No broken links found",
+ "empty_description": "None of your saved links currently have a reported crawl error.",
+ "select_visible": "Select all visible links",
+ "bookmark": "Bookmark",
+ "issue": "Issue",
+ "never_crawled": "No crawl date",
+ "issue_blocked": "Access blocked",
+ "issue_missing": "Not found",
+ "issue_server": "Server error",
+ "issue_http": "HTTP error",
+ "issue_failed": "Crawl failed",
+ "issue_blocked_hint": "The site restricted this request",
+ "issue_missing_hint": "The page may have been removed",
+ "issue_server_hint": "The site could not complete the request",
+ "issue_http_hint": "The site returned an unexpected response",
+ "issue_failed_hint": "No successful response was recorded",
+ "queued": "Queued",
+ "queued_count_one": "{{count}} link queued for another crawl.",
+ "queued_count_other": "{{count}} links queued for another crawl.",
+ "deleted_count_one": "{{count}} bookmark deleted.",
+ "deleted_count_other": "{{count}} bookmarks deleted.",
+ "action_failed": "{{count}} of {{total}} failed.",
+ "delete_title_one": "Delete {{count}} bookmark?",
+ "delete_title_other": "Delete {{count}} bookmarks?",
+ "delete_description": "This permanently deletes the selected bookmarks and their saved files, highlights and notes. This cannot be undone."
},
"manage_assets": {
"manage_assets": "Manage Assets",
@@ -629,6 +671,19 @@
"backup_queued": "Backup job has been queued! It will be processed shortly.",
"backup_deleted": "Backup has been deleted!"
}
+ },
+ "maintenance": {
+ "clear_filters": "Clear filters",
+ "clear_selection": "Clear selection",
+ "delete_selected": "Delete selected",
+ "try_again": "Try again",
+ "sort": "Sort order",
+ "no_matches": "No matches found",
+ "adjust_filters": "Try a different search or clear your filters.",
+ "select_item": "Select {{name}}",
+ "selected": "{{count}} selected",
+ "showing": "Showing {{shown}} of {{total}}",
+ "load_more": "Load more"
}
},
"admin": {
diff --git a/apps/web/lib/i18n/locales/en_US/translation.json b/apps/web/lib/i18n/locales/en_US/translation.json
index 776f96d3..15dc8a41 100644
--- a/apps/web/lib/i18n/locales/en_US/translation.json
+++ b/apps/web/lib/i18n/locales/en_US/translation.json
@@ -463,7 +463,49 @@
"broken_links": "Broken Links",
"last_crawled_at": "Last Crawled At",
"crawling_status": "Crawling Status",
- "crawling_failed": "Crawling Failed"
+ "crawling_failed": "Crawling Failed",
+ "description": "Review links that could not be fetched, retry them or remove old bookmarks.",
+ "category_all": "Needs attention",
+ "category_blocked": "Access blocked",
+ "category_missing": "Not found",
+ "category_other": "Other errors",
+ "category_all_hint": "Links to review",
+ "category_blocked_hint": "Restricted or rate limited",
+ "category_missing_hint": "Page missing or removed",
+ "category_other_hint": "Server or crawl failures",
+ "blocked_hint": "A blocked request doesn't always mean a broken link. Some sites restrict automated access. Open the original page to check before deleting.",
+ "search": "Search by title or URL…",
+ "sort_recent": "Recently checked",
+ "sort_oldest": "Oldest check first",
+ "sort_domain": "Website A–Z",
+ "result_count_one": "{{count}} link to review",
+ "result_count_other": "{{count}} links to review",
+ "retry_selected": "Retry selected",
+ "empty": "No broken links found",
+ "empty_description": "None of your saved links currently have a reported crawl error.",
+ "select_visible": "Select all visible links",
+ "bookmark": "Bookmark",
+ "issue": "Issue",
+ "never_crawled": "No crawl date",
+ "issue_blocked": "Access blocked",
+ "issue_missing": "Not found",
+ "issue_server": "Server error",
+ "issue_http": "HTTP error",
+ "issue_failed": "Crawl failed",
+ "issue_blocked_hint": "The site restricted this request",
+ "issue_missing_hint": "The page may have been removed",
+ "issue_server_hint": "The site could not complete the request",
+ "issue_http_hint": "The site returned an unexpected response",
+ "issue_failed_hint": "No successful response was recorded",
+ "queued": "Queued",
+ "queued_count_one": "{{count}} link queued for another crawl.",
+ "queued_count_other": "{{count}} links queued for another crawl.",
+ "deleted_count_one": "{{count}} bookmark deleted.",
+ "deleted_count_other": "{{count}} bookmarks deleted.",
+ "action_failed": "{{count}} of {{total}} failed.",
+ "delete_title_one": "Delete {{count}} bookmark?",
+ "delete_title_other": "Delete {{count}} bookmarks?",
+ "delete_description": "This permanently deletes the selected bookmarks and their saved files, highlights and notes. This cannot be undone."
},
"manage_assets": {
"manage_assets": "Manage Assets",
@@ -720,6 +762,19 @@
"backup_queued": "Backup job has been queued! It will be processed shortly.",
"backup_deleted": "Backup has been deleted!"
}
+ },
+ "maintenance": {
+ "clear_filters": "Clear filters",
+ "clear_selection": "Clear selection",
+ "delete_selected": "Delete selected",
+ "try_again": "Try again",
+ "sort": "Sort order",
+ "no_matches": "No matches found",
+ "adjust_filters": "Try a different search or clear your filters.",
+ "select_item": "Select {{name}}",
+ "selected": "{{count}} selected",
+ "showing": "Showing {{shown}} of {{total}}",
+ "load_more": "Load more"
}
},
"admin": {
diff --git a/packages/trpc/routers/bookmarks.test.ts b/packages/trpc/routers/bookmarks.test.ts
index 15c9efa2..233f54fd 100644
--- a/packages/trpc/routers/bookmarks.test.ts
+++ b/packages/trpc/routers/bookmarks.test.ts
@@ -26,6 +26,9 @@ vi.mock("@karakeep/shared-server", async (original) => {
LinkCrawlerQueue: {
enqueue: vi.fn(),
},
+ LowPriorityCrawlerQueue: {
+ enqueue: vi.fn(),
+ },
OpenAIQueue: {
enqueue: vi.fn(),
},
@@ -1098,6 +1101,7 @@ describe("Bookmark Routes", () => {
const brokenBookmark = await api.createBookmark({
url: "https://broken-link.com",
type: BookmarkTypes.LINK,
+ title: "Broken page",
});
await db
.update(bookmarkLinks)
@@ -1111,11 +1115,26 @@ describe("Bookmark Routes", () => {
).toBeTruthy();
expect(result.bookmarks[0].url).toEqual("https://broken-link.com");
expect(result.bookmarks[0].isCrawlingFailure).toBeTruthy();
+ expect(result.bookmarks[0].title).toBe("Broken page");
+ expect(result.bookmarks[0].isCrawling).toBe(false);
+ expect((await apiCallers[1].bookmarks.getBrokenLinks()).bookmarks).toEqual(
+ [],
+ );
+ await db
+ .update(bookmarkLinks)
+ .set({ crawlStatus: "failure", crawlStatusCode: 403 })
+ .where(eq(bookmarkLinks.id, brokenBookmark.id));
+ await api.recrawlBookmark({ bookmarkId: brokenBookmark.id });
+ expect((await api.getBrokenLinks()).bookmarks[0]).toMatchObject({
+ isCrawling: true,
+ isCrawlingFailure: false,
+ statusCode: 403,
+ });
// Test with no broken links
await db
.update(bookmarkLinks)
- .set({ crawlStatus: "success" })
+ .set({ crawlStatus: "success", crawlStatusCode: 200 })
.where(eq(bookmarkLinks.id, brokenBookmark.id));
const emptyResult = await api.getBrokenLinks();
expect(emptyResult.bookmarks.length).toEqual(0);
diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts
index 7269560b..058c7f37 100644
--- a/packages/trpc/routers/bookmarks.ts
+++ b/packages/trpc/routers/bookmarks.ts
@@ -843,6 +843,10 @@ export const bookmarksAppRouter = router({
priority: QueuePriority.Low,
idempotencyKey: buildCrawlIdempotencyKey(payload),
});
+ await ctx.db
+ .update(bookmarkLinks)
+ .set({ crawlStatus: "pending" })
+ .where(eq(bookmarkLinks.id, input.bookmarkId));
}),
updateReadingProgress: bookmarksProcedure
.input(
@@ -1442,6 +1446,8 @@ export const bookmarksAppRouter = router({
z.object({
id: z.string(),
url: z.string(),
+ title: z.string().nullable(),
+ isCrawling: z.boolean(),
statusCode: z.number().nullable(),
isCrawlingFailure: z.boolean(),
crawledAt: z.date().nullable(),
@@ -1455,6 +1461,8 @@ export const bookmarksAppRouter = router({
.select({
id: bookmarkLinks.id,
url: bookmarkLinks.url,
+ userTitle: bookmarks.title,
+ linkTitle: bookmarkLinks.title,
crawlStatusCode: bookmarkLinks.crawlStatusCode,
crawlingStatus: bookmarkLinks.crawlStatus,
crawledAt: bookmarkLinks.crawledAt,
@@ -1476,6 +1484,8 @@ export const bookmarksAppRouter = router({
bookmarks: brokenLinkBookmarks.map((b) => ({
id: b.id,
url: b.url,
+ title: b.userTitle || b.linkTitle || null,
+ isCrawling: b.crawlingStatus === "pending",
statusCode: b.crawlStatusCode,
isCrawlingFailure: b.crawlingStatus === "failure",
crawledAt: b.crawledAt,