Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

feat(mobile): implement full offline storage and sync functionality for the preserved formats

daniel31x13f6c512cce2d0
diff --git a/apps/mobile/app/(tabs)/settings/index.tsx b/apps/mobile/app/(tabs)/settings/index.tsx
index 6c1cd7f6..b6a2cb15 100644
--- a/apps/mobile/app/(tabs)/settings/index.tsx
+++ b/apps/mobile/app/(tabs)/settings/index.tsx
@@ -8,6 +8,8 @@ import {
TouchableOpacity,
Platform,
Alert,
+ Switch,
+ ActivityIndicator,
} from "react-native";
import { nativeApplicationVersion } from "expo-application";
import { useColorScheme } from "nativewind";
@@ -17,14 +19,24 @@ import {
AppWindowMac,
Check,
ChevronRight,
+ Download,
ExternalLink,
Folder,
+ HardDrive,
LogOut,
Mail,
Moon,
+ RefreshCw,
Smartphone,
Sun,
+ Trash2,
} from "lucide-react-native";
+import { clearCache } from "@/lib/cache";
+import {
+ formatBytes,
+ recomputeStorage,
+ useOfflineSyncStore,
+} from "@/lib/offlineSync";
import useDataStore from "@/store/data";
import * as Clipboard from "expo-clipboard";
import { useRouter } from "expo-router";
@@ -45,6 +57,21 @@ export default function SettingsScreen() {
const router = useRouter();
+ const syncStatus = useOfflineSyncStore((s) => s.status);
+ const syncProcessed = useOfflineSyncStore((s) => s.processed);
+ const syncTotal = useOfflineSyncStore((s) => s.total);
+ const bytesUsed = useOfflineSyncStore((s) => s.bytesUsed);
+ const syncPercent =
+ syncTotal > 0 ? Math.floor((syncProcessed / syncTotal) * 100) : null;
+ const syncStatusLabel =
+ syncStatus === "paused"
+ ? "Waiting for connection"
+ : syncPercent === null
+ ? "Preparing…"
+ : syncPercent >= 100
+ ? "Up to date"
+ : `${syncPercent}%`;
+
return (
<View
style={styles.container}
@@ -228,6 +255,111 @@ export default function SettingsScreen() {
</View>
</View>
+ <View>
+ <Text className="mb-4 mx-4 text-neutral">Offline Storage</Text>
+ <View className="bg-base-200 rounded-xl flex-col">
+ <View className="py-3 px-4">
+ <View className="flex-row gap-2 items-center justify-between">
+ <View className="flex-row items-center gap-2">
+ <Download
+ size={20}
+ color={rawTheme[colorScheme as ThemeName].neutral}
+ />
+ <Text className="text-base-content">
+ Save for offline access
+ </Text>
+ </View>
+ <Switch
+ value={!!data.offlineEnabled}
+ onValueChange={(value) =>
+ updateData({ offlineEnabled: value })
+ }
+ trackColor={{
+ true: rawTheme[colorScheme as ThemeName].primary,
+ }}
+ thumbColor={rawTheme[colorScheme as ThemeName]["base-100"]}
+ />
+ </View>
+ <Text className="text-sm text-neutral mt-2 ml-7">
+ Automatically saves preserved formats for links loaded in the
+ app to this device for offline access. When this is off, only
+ formats you open are saved.
+ </Text>
+ </View>
+ {data.offlineEnabled ? (
+ <>
+ <View className="h-px bg-neutral-content ml-12" />
+ <View className="flex-row gap-2 items-center justify-between py-3 px-4">
+ <View className="flex-row items-center gap-2">
+ <RefreshCw
+ size={20}
+ color={rawTheme[colorScheme as ThemeName].neutral}
+ />
+ <Text className="text-base-content">Offline sync</Text>
+ </View>
+ <View className="flex-row items-center gap-2">
+ {syncStatus === "syncing" ? (
+ <ActivityIndicator
+ size="small"
+ color={rawTheme[colorScheme as ThemeName].primary}
+ />
+ ) : null}
+ <Text className="text-neutral">{syncStatusLabel}</Text>
+ </View>
+ </View>
+ </>
+ ) : null}
+ <View className="h-px bg-neutral-content ml-12" />
+ <View className="flex-row gap-2 items-center justify-between py-3 px-4">
+ <View className="flex-row items-center gap-2">
+ <HardDrive
+ size={20}
+ color={rawTheme[colorScheme as ThemeName].neutral}
+ />
+ <Text className="text-base-content">Storage used</Text>
+ </View>
+ <Text className="text-neutral">{formatBytes(bytesUsed)}</Text>
+ </View>
+ <View className="h-px bg-neutral-content ml-12" />
+ <TouchableOpacity
+ className="flex-row gap-2 items-center justify-between py-3 px-4"
+ disabled={syncStatus === "syncing"}
+ onPress={() =>
+ Alert.alert(
+ "Clear cache",
+ "This will delete all downloaded formats and previews from this device.",
+ [
+ { text: "Cancel", style: "cancel" },
+ {
+ text: "Clear",
+ style: "destructive",
+ onPress: async () => {
+ await clearCache();
+ useOfflineSyncStore.getState().reset();
+ await recomputeStorage();
+ },
+ },
+ ]
+ )
+ }
+ >
+ <View className="flex-row items-center gap-2">
+ <Trash2
+ size={20}
+ color={syncStatus === "syncing" ? "gray" : "red"}
+ />
+ <Text
+ className={
+ syncStatus === "syncing" ? "text-neutral" : "text-red-500"
+ }
+ >
+ Clear cache
+ </Text>
+ </View>
+ </TouchableOpacity>
+ </View>
+ </View>
+
<View>
<Text className="mb-4 mx-4 text-neutral">Contact Us</Text>
<View className="bg-base-200 rounded-xl flex-col">
diff --git a/apps/mobile/app/(tabs)/tags/index.tsx b/apps/mobile/app/(tabs)/tags/index.tsx
index 151e0a93..bfe1e745 100644
--- a/apps/mobile/app/(tabs)/tags/index.tsx
+++ b/apps/mobile/app/(tabs)/tags/index.tsx
@@ -16,6 +16,8 @@ import { useColorScheme } from "nativewind";
import { TagSort } from "@linkwarden/types/global";
import { useTags } from "@linkwarden/router/tags";
import { isAtLeastInstanceVersion, useConfig } from "@linkwarden/router/config";
+import { useQueryClient } from "@tanstack/react-query";
+import { resetInfiniteQueryPagination } from "@linkwarden/router/lib";
const MIN_TAG_SEARCH_VERSION = "2.14.1";
@@ -34,6 +36,7 @@ export default function TagsScreen() {
const { auth } = useAuthStore();
const { search } = useLocalSearchParams<{ search?: string }>();
const config = useConfig(auth);
+ const queryClient = useQueryClient();
const searchQuery = decodeSearchParam(search);
const supportsTagSearch = isAtLeastInstanceVersion(
config.data?.INSTANCE_VERSION,
@@ -72,7 +75,9 @@ export default function TagsScreen() {
refreshControl={
<Spinner
refreshing={tags.isRefetching}
- onRefresh={() => tags.refetch()}
+ onRefresh={() =>
+ resetInfiniteQueryPagination(queryClient, ["tags"])
+ }
progressBackgroundColor={
rawTheme[colorScheme as ThemeName]["base-200"]
}
diff --git a/apps/mobile/app/_layout.tsx b/apps/mobile/app/_layout.tsx
index 3266a827..36df19bc 100644
--- a/apps/mobile/app/_layout.tsx
+++ b/apps/mobile/app/_layout.tsx
@@ -22,6 +22,7 @@ import { KeyboardProvider } from "react-native-keyboard-controller";
import { queryClient } from "@/lib/queryClient";
import { StatusBar } from "expo-status-bar";
import * as Sentry from "@sentry/react-native";
+import OfflineSyncProvider from "@/components/OfflineSyncProvider";
Sentry.init({
dsn: "https://00d7eed9e810cbbf91a7ed3547e37100@o4510998442475520.ingest.us.sentry.io/4511033679609856",
@@ -90,7 +91,7 @@ export default Sentry.wrap(function RootLayout() {
maxAge: Infinity,
dehydrateOptions: {
shouldDehydrateMutation: () => true,
- shouldDehydrateQuery: () => true,
+ shouldDehydrateQuery: (query) => query.state.status === "success",
},
}}
onSuccess={() => {
@@ -112,6 +113,7 @@ const RootComponent = ({ isLoading }: { isLoading: boolean }) => {
style={[{ flex: 1 }, colorScheme === "dark" ? darkTheme : lightTheme]}
>
<SheetProvider>
+ <OfflineSyncProvider />
<StatusBar
style={colorScheme === "dark" ? "light" : "dark"}
backgroundColor={rawTheme[colorScheme as ThemeName]["base-100"]}
diff --git a/apps/mobile/components/ActionSheets/LinkDetailsSheet.tsx b/apps/mobile/components/ActionSheets/LinkDetailsSheet.tsx
index 43e9e79f..22fde5c8 100644
--- a/apps/mobile/components/ActionSheets/LinkDetailsSheet.tsx
+++ b/apps/mobile/components/ActionSheets/LinkDetailsSheet.tsx
@@ -1,7 +1,6 @@
import { useEffect, useState, type ReactNode } from "react";
import {
ActivityIndicator,
- Image,
Linking,
Text,
TouchableOpacity,
@@ -14,14 +13,12 @@ import ActionSheet, {
SheetProps,
} from "react-native-actions-sheet";
import { decode } from "html-entities";
-import * as Clipboard from "expo-clipboard";
import { useRouter } from "expo-router";
import { useColorScheme } from "nativewind";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import {
CalendarDays,
ChevronRight,
- Copy,
ExternalLink,
File,
FileCode,
@@ -360,7 +357,10 @@ export default function LinkDetailsSheet(
<View className="rounded-lg bg-base-100 px-4 py-3">
{link.description ? (
- <Text className="text-base leading-6 text-base-content">
+ <Text
+ className="text-base leading-6 text-base-content"
+ selectable
+ >
{decode(link.description)}
</Text>
) : (
diff --git a/apps/mobile/components/Links.tsx b/apps/mobile/components/Links.tsx
index b08eb219..95ae3b16 100644
--- a/apps/mobile/components/Links.tsx
+++ b/apps/mobile/components/Links.tsx
@@ -11,6 +11,8 @@ import { LinkIncludingShortenedCollectionAndTags } from "@linkwarden/types/globa
import Spinner from "@/components/ui/Spinner";
import { rawTheme, ThemeName } from "@/lib/colors";
import { useColorScheme } from "nativewind";
+import { useQueryClient } from "@tanstack/react-query";
+import { resetInfiniteQueryPagination } from "@linkwarden/router/lib";
const RenderItem = React.memo(
({ item }: { item: LinkIncludingShortenedCollectionAndTags }) => {
@@ -25,6 +27,7 @@ type Props = {
export default function Links({ links, data }: Props) {
const { colorScheme } = useColorScheme();
+ const queryClient = useQueryClient();
const [promptedRefetch, setPromptedRefetch] = useState(false);
return data.isLoading ? (
@@ -42,7 +45,7 @@ export default function Links({ links, data }: Props) {
refreshing={data.isRefetching && promptedRefetch}
onRefresh={async () => {
setPromptedRefetch(true);
- await data.refetch();
+ await resetInfiniteQueryPagination(queryClient, ["links"]);
setPromptedRefetch(false);
}}
progressBackgroundColor={
diff --git a/apps/mobile/components/OfflineSyncProvider.tsx b/apps/mobile/components/OfflineSyncProvider.tsx
new file mode 100644
index 00000000..8ceb6b42
--- /dev/null
+++ b/apps/mobile/components/OfflineSyncProvider.tsx
@@ -0,0 +1,56 @@
+import { useEffect } from "react";
+import { useConfig } from "@linkwarden/router/config";
+import useAuthStore from "@/store/auth";
+import useDataStore from "@/store/data";
+import {
+ recomputeStorage,
+ startSync,
+ stopSync,
+ subscribeToConnectivity,
+ subscribeToQueryCache,
+ unsubscribeFromConnectivity,
+ unsubscribeFromQueryCache,
+} from "@/lib/offlineSync";
+
+export default function OfflineSyncProvider() {
+ const { auth } = useAuthStore();
+ const { data } = useDataStore();
+ const config = useConfig(auth);
+
+ useEffect(() => {
+ subscribeToConnectivity();
+ recomputeStorage();
+
+ return () => unsubscribeFromConnectivity();
+ }, []);
+
+ useEffect(() => {
+ if (auth.status !== "authenticated") {
+ stopSync();
+ unsubscribeFromQueryCache();
+ return;
+ }
+
+ if (!data.offlineEnabled) {
+ stopSync();
+ unsubscribeFromQueryCache();
+ return;
+ }
+
+ if (config.isLoading) return;
+
+ startSync(auth, config.data?.USER_CONTENT_DOMAIN);
+ subscribeToQueryCache();
+
+ return () => stopSync();
+ }, [
+ auth.status,
+ auth.instance,
+ auth.session,
+ data.offlineEnabled,
+ config.isLoading,
+ config.data?.USER_CONTENT_DOMAIN,
+ ]);
+
+ return null;
+}
diff --git a/apps/mobile/lib/cache.ts b/apps/mobile/lib/cache.ts
index 6b9b01d7..3f674e61 100644
--- a/apps/mobile/lib/cache.ts
+++ b/apps/mobile/lib/cache.ts
@@ -1,5 +1,7 @@
import * as FileSystem from "expo-file-system/legacy";
import NetInfo from "@react-native-community/netinfo";
+import { ArchivedFormat, MobileAuth } from "@linkwarden/types/global";
+import getPreservedFormatUrl from "@linkwarden/lib/getPreservedFormatUrl";
type LoadCacheOrFetchOptions<T> = {
filePath: string;
@@ -55,6 +57,153 @@ export const loadCacheOrFetch = async <T = string>({
}
};
+export type CacheFormat =
+ | "readable"
+ | "webpage"
+ | "jpeg"
+ | "png"
+ | "pdf"
+ | "preview";
+
+export const getCachePathForFormat = (
+ linkId: number,
+ format: CacheFormat
+): string => {
+ switch (format) {
+ case "readable":
+ return (
+ FileSystem.documentDirectory +
+ `archivedData/readable/link_${linkId}.html`
+ );
+ case "webpage":
+ return (
+ FileSystem.documentDirectory +
+ `archivedData/webpage/link_${linkId}.html`
+ );
+ case "jpeg":
+ return (
+ FileSystem.documentDirectory + `archivedData/jpeg/link_${linkId}.jpeg`
+ );
+ case "png":
+ return (
+ FileSystem.documentDirectory + `archivedData/png/link_${linkId}.png`
+ );
+ case "pdf":
+ return (
+ FileSystem.documentDirectory + `archivedData/pdf/link_${linkId}.pdf`
+ );
+ case "preview":
+ return (
+ FileSystem.documentDirectory +
+ `archivedData/previews/link_${linkId}.jpg`
+ );
+ }
+};
+
+type FetchFormatToCacheOptions = {
+ link: { id: number; updatedAt?: string | Date | null };
+ format: CacheFormat;
+ auth: MobileAuth;
+ userContentDomain?: string | null;
+};
+
+export const fetchFormatToCache = async ({
+ link,
+ format,
+ auth,
+ userContentDomain,
+}: FetchFormatToCacheOptions): Promise<{
+ uri: string;
+ size: number;
+ delta: number;
+}> => {
+ const filePath = getCachePathForFormat(link.id, format);
+
+ await FileSystem.makeDirectoryAsync(
+ filePath.substring(0, filePath.lastIndexOf("/")),
+ { intermediates: true }
+ ).catch(() => {});
+
+ const prev = await FileSystem.getInfoAsync(filePath);
+ const prevSize = prev.exists ? (prev as any).size ?? 0 : 0;
+
+ const archivedFormat =
+ format === "readable"
+ ? ArchivedFormat.readability
+ : format === "webpage"
+ ? ArchivedFormat.monolith
+ : format === "jpeg"
+ ? ArchivedFormat.jpeg
+ : format === "png"
+ ? ArchivedFormat.png
+ : format === "pdf"
+ ? ArchivedFormat.pdf
+ : ArchivedFormat.jpeg;
+
+ const previewSuffix =
+ format === "preview" ? `&preview=true&updatedAt=${link.updatedAt}` : "";
+
+ let apiUrl: string;
+ let headers: Record<string, string> | undefined = {
+ Authorization: `Bearer ${auth.session}`,
+ };
+
+ if (format === "webpage" && userContentDomain) {
+ apiUrl = await getPreservedFormatUrl({
+ tokenEndpoint: `${auth.instance}/api/v1/preserved/token`,
+ linkId: link.id,
+ format: archivedFormat,
+ headers,
+ });
+ headers = undefined;
+ } else {
+ apiUrl = `${auth.instance}/api/v1/archives/${link.id}?format=${archivedFormat}${previewSuffix}`;
+ }
+
+ const tmpPath = `${filePath}.part`;
+ await FileSystem.deleteAsync(tmpPath, { idempotent: true }).catch(() => {});
+
+ try {
+ if (format === "readable") {
+ const response = await fetch(apiUrl, { headers });
+ if (!response.ok) throw new Error(`HTTP ${response.status}`);
+ const data = (await response.json()).content;
+
+ await FileSystem.writeAsStringAsync(tmpPath, data, {
+ encoding: FileSystem.EncodingType.UTF8,
+ });
+ } else {
+ const result = await FileSystem.downloadAsync(apiUrl, tmpPath, {
+ headers,
+ });
+ if (result.status < 200 || result.status >= 300) {
+ throw new Error(`HTTP ${result.status}`);
+ }
+ }
+
+ await FileSystem.deleteAsync(filePath, { idempotent: true }).catch(
+ () => {}
+ );
+ await FileSystem.moveAsync({ from: tmpPath, to: filePath });
+
+ const info = await FileSystem.getInfoAsync(filePath);
+ const size = (info as any).size ?? 0;
+ return { uri: filePath, size, delta: size - prevSize };
+ } catch (e) {
+ await FileSystem.deleteAsync(tmpPath, { idempotent: true }).catch(() => {});
+ throw e;
+ }
+};
+
+export const getCacheSize = async (): Promise<number> => {
+ const root = FileSystem.documentDirectory + "archivedData";
+
+ const rootInfo = await FileSystem.getInfoAsync(root);
+ if (!rootInfo.exists) return 0;
+
+ return rootInfo.size ?? 0;
+};
+
export const clearCache = async () => {
await Promise.all([
FileSystem.deleteAsync(FileSystem.documentDirectory + "archivedData", {
@@ -67,25 +216,13 @@ export const clearCache = async () => {
};
export const deleteLinkCache = async (linkId: number) => {
- const readablePath =
- FileSystem.documentDirectory + `archivedData/readable/link_${linkId}.html`;
- const webpagePath =
- FileSystem.documentDirectory + `archivedData/webpage/link_${linkId}.html`;
- const jpegPath =
- FileSystem.documentDirectory + `archivedData/jpeg/link_${linkId}.jpeg`;
- const pngPath =
- FileSystem.documentDirectory + `archivedData/png/link_${linkId}.png`;
- const pdfPath =
- FileSystem.documentDirectory + `archivedData/pdf/link_${linkId}.pdf`;
- const previewPath =
- FileSystem.documentDirectory + `archivedData/previews/link_${linkId}.jpg`;
-
- await Promise.all([
- FileSystem.deleteAsync(readablePath, { idempotent: true }),
- FileSystem.deleteAsync(webpagePath, { idempotent: true }),
- FileSystem.deleteAsync(jpegPath, { idempotent: true }),
- FileSystem.deleteAsync(pngPath, { idempotent: true }),
- FileSystem.deleteAsync(pdfPath, { idempotent: true }),
- FileSystem.deleteAsync(previewPath, { idempotent: true }),
- ]);
+ await Promise.all(
+ (
+ ["readable", "webpage", "jpeg", "png", "pdf", "preview"] as CacheFormat[]
+ ).map((format) =>
+ FileSystem.deleteAsync(getCachePathForFormat(linkId, format), {
+ idempotent: true,
+ })
+ )
+ );
};
diff --git a/apps/mobile/lib/offlineSync.ts b/apps/mobile/lib/offlineSync.ts
new file mode 100644
index 00000000..91f485ed
--- /dev/null
+++ b/apps/mobile/lib/offlineSync.ts
@@ -0,0 +1,578 @@
+import { create } from "zustand";
+import NetInfo from "@react-native-community/netinfo";
+import {
+ LinkIncludingShortenedCollectionAndTags,
+ MobileAuth,
+} from "@linkwarden/types/global";
+import { formatAvailable } from "@linkwarden/lib/formatStats";
+import * as FileSystem from "expo-file-system/legacy";
+import { CacheFormat, fetchFormatToCache, getCacheSize } from "@/lib/cache";
+import { queryClient } from "@/lib/queryClient";
+
+type SyncStatus = "idle" | "syncing" | "paused";
+
+type OfflineSyncState = {
+ status: SyncStatus;
+ processed: number;
+ total: number;
+ failed: number;
+ currentLinkId: number | null;
+ bytesUsed: number;
+ setStatus: (status: SyncStatus) => void;
+ setProgress: (processed: number, total: number) => void;
+ incrementProcessed: () => void;
+ incrementFailed: () => void;
+ setCurrentLinkId: (id: number | null) => void;
+ setBytesUsed: (bytes: number) => void;
+ addBytes: (delta: number) => void;
+ reset: () => void;
+};
+
+export const useOfflineSyncStore = create<OfflineSyncState>((set) => ({
+ status: "idle",
+ processed: 0,
+ total: 0,
+ failed: 0,
+ currentLinkId: null,
+ bytesUsed: 0,
+ setStatus: (status) => set({ status }),
+ setProgress: (processed, total) =>
+ set((s) => ({ processed, total, failed: processed === 0 ? 0 : s.failed })),
+ incrementProcessed: () => set((s) => ({ processed: s.processed + 1 })),
+ incrementFailed: () => set((s) => ({ failed: s.failed + 1 })),
+ setCurrentLinkId: (id) => set({ currentLinkId: id }),
+ setBytesUsed: (bytes) => set({ bytesUsed: bytes }),
+ addBytes: (delta) =>
+ set((s) => ({ bytesUsed: Math.max(0, s.bytesUsed + delta) })),
+ reset: () =>
+ set({
+ status: "idle",
+ processed: 0,
+ total: 0,
+ failed: 0,
+ currentLinkId: null,
+ }),
+}));
+
+let cancelled = false;
+let runPromise: Promise<void> | null = null;
+let syncGeneration = 0;
+let netUnsubscribe: (() => void) | null = null;
+let cacheUnsubscribe: (() => void) | null = null;
+let cacheSyncTimeout: ReturnType<typeof setTimeout> | null = null;
+let activeAuth: MobileAuth | null = null;
+let activeUserContentDomain: string | null | undefined = undefined;
+let syncQueryWriteDepth = 0;
+let ignoredQueryKeyClearTimeout: ReturnType<typeof setTimeout> | null = null;
+
+const archiveRoot = FileSystem.documentDirectory + "archivedData";
+const staleFileGraceMs = 5000;
+
+const formatDirectories: Record<
+ CacheFormat,
+ { dirName: string; extension: string }
+> = {
+ readable: { dirName: "readable", extension: "html" },
+ webpage: { dirName: "webpage", extension: "html" },
+ jpeg: { dirName: "jpeg", extension: "jpeg" },
+ png: { dirName: "png", extension: "png" },
+ pdf: { dirName: "pdf", extension: "pdf" },
+ preview: { dirName: "previews", extension: "jpg" },
+};
+
+type CachedFile = {
+ path: string;
+ size: number;
+ linkId?: number;
+ format?: CacheFormat;
+ modifiedAt?: number;
+ partial: boolean;
+};
+
+type CacheSnapshot = {
+ bytesUsed: number;
+ filesByLinkId: Map<number, CachedFile[]>;
+ formatsByLinkId: Map<number, Map<CacheFormat, CachedFile>>;
+ partialFiles: CachedFile[];
+};
+
+type LinksCacheSnapshot = {
+ links: LinkIncludingShortenedCollectionAndTags[];
+ hasLinkSource: boolean;
+};
+
+const getLinkFreshnessTime = (link: LinkIncludingShortenedCollectionAndTags) =>
+ new Date((link as any)?.updatedAt).getTime();
+
+const getFileModifiedAt = (info: any): number | undefined => {
+ if (
+ typeof info.modificationTime !== "number" ||
+ !Number.isFinite(info.modificationTime)
+ ) {
+ return undefined;
+ }
+
+ return info.modificationTime > 10_000_000_000
+ ? info.modificationTime
+ : info.modificationTime * 1000;
+};
+
+const isOnline = (state: {
+ isConnected: boolean | null;
+ isInternetReachable?: boolean | null;
+}) => state.isConnected === true && state.isInternetReachable !== false;
+
+const ignoredQueryKeySignatures = new Set<string>();
+
+const getQueryKeySignature = (queryKey: unknown) => {
+ try {
+ return JSON.stringify(queryKey);
+ } catch {
+ return undefined;
+ }
+};
+
+const ignoreQueryKeyTemporarily = (queryKey: unknown) => {
+ const signature = getQueryKeySignature(queryKey);
+ if (!signature) return;
+
+ ignoredQueryKeySignatures.add(signature);
+ if (ignoredQueryKeyClearTimeout) clearTimeout(ignoredQueryKeyClearTimeout);
+ ignoredQueryKeyClearTimeout = setTimeout(() => {
+ ignoredQueryKeySignatures.clear();
+ ignoredQueryKeyClearTimeout = null;
+ }, 500);
+};
+
+const sameArchiveState = (a: any, b: LinkIncludingShortenedCollectionAndTags) =>
+ a?.id === b.id &&
+ a?.updatedAt === b.updatedAt &&
+ a?.preview === b.preview &&
+ a?.readable === (b as any).readable &&
+ a?.monolith === (b as any).monolith &&
+ a?.pdf === (b as any).pdf &&
+ a?.image === (b as any).image;
+
+const setCachedLinkQueryData = (
+ link: LinkIncludingShortenedCollectionAndTags & { id: number }
+) => {
+ const queryKey = ["link", link.id, false] as const;
+ const existing = queryClient.getQueryData(queryKey);
+ if (sameArchiveState(existing, link)) return;
+
+ ignoreQueryKeyTemporarily(queryKey);
+ syncQueryWriteDepth += 1;
+ try {
+ queryClient.setQueryData(queryKey, link);
+ } finally {
+ syncQueryWriteDepth -= 1;
+ }
+};
+
+const getLinksFromCache = (): LinksCacheSnapshot => {
+ const linksById = new Map<number, LinkIncludingShortenedCollectionAndTags>();
+ const add = (link: any) => {
+ const id = Number(link?.id);
+ if (!Number.isFinite(id)) return;
+
+ const existing = linksById.get(id);
+ if (
+ !existing ||
+ getLinkFreshnessTime(link) >= getLinkFreshnessTime(existing)
+ ) {
+ linksById.set(id, link);
+ }
+ };
+ let hasLinkSource = false;
+
+ for (const query of queryClient.getQueryCache().getAll()) {
+ const key = query.queryKey;
+ const data = query.state.data as any;
+ if (!data) continue;
+ if (!Array.isArray(key)) continue;
+
+ if (key[0] === "links") {
+ if (Array.isArray(data.pages)) hasLinkSource = true;
+ for (const page of data.pages ?? []) {
+ for (const link of page?.links ?? []) add(link);
+ }
+ } else if (key[0] === "dashboardData") {
+ if (Array.isArray(data.links)) hasLinkSource = true;
+ for (const link of data.links ?? []) add(link);
+ } else if (key[0] === "link") {
+ if (Number.isFinite(Number(data?.id))) hasLinkSource = true;
+ add(data);
+ }
+ }
+
+ return {
+ links: Array.from(linksById.values()),
+ hasLinkSource,
+ };
+};
+
+const getFormatForCacheFile = (
+ dirName: string,
+ extension: string
+): CacheFormat | undefined => {
+ const entry = Object.entries(formatDirectories).find(
+ ([, value]) => value.dirName === dirName && value.extension === extension
+ );
+ return entry?.[0] as CacheFormat | undefined;
+};
+
+const parseCacheFile = (
+ dirName: string,
+ fileName: string
+): Pick<CachedFile, "linkId" | "format" | "partial"> => {
+ const match = fileName.match(/^link_(\d+)\.([a-z0-9]+)(\.part)?$/i);
+ if (!match) return { partial: false };
+
+ const linkId = Number(match[1]);
+ if (!Number.isFinite(linkId)) return { partial: false };
+
+ return {
+ linkId,
+ format: getFormatForCacheFile(dirName, match[2].toLowerCase()),
+ partial: Boolean(match[3]),
+ };
+};
+
+const createEmptyCacheSnapshot = (): CacheSnapshot => ({
+ bytesUsed: 0,
+ filesByLinkId: new Map(),
+ formatsByLinkId: new Map(),
+ partialFiles: [],
+});
+
+const addFileToSnapshot = (snapshot: CacheSnapshot, file: CachedFile) => {
+ snapshot.bytesUsed += file.size;
+
+ if (file.partial) {
+ snapshot.partialFiles.push(file);
+ }
+
+ if (file.linkId === undefined) return;
+
+ const files = snapshot.filesByLinkId.get(file.linkId) ?? [];
+ files.push(file);
+ snapshot.filesByLinkId.set(file.linkId, files);
+
+ if (!file.format || file.partial) return;
+
+ const formats = snapshot.formatsByLinkId.get(file.linkId) ?? new Map();
+ formats.set(file.format, file);
+ snapshot.formatsByLinkId.set(file.linkId, formats);
+};
+
+const scanCacheSnapshot = async (): Promise<CacheSnapshot> => {
+ const snapshot = createEmptyCacheSnapshot();
+ const rootInfo = await FileSystem.getInfoAsync(archiveRoot).catch(() => null);
+ if (!rootInfo?.exists) return snapshot;
+
+ const subdirs = await FileSystem.readDirectoryAsync(archiveRoot).catch(
+ () => []
+ );
+
+ await Promise.all(
+ subdirs.map(async (dirName) => {
+ const dirPath = `${archiveRoot}/${dirName}`;
+ const files = await FileSystem.readDirectoryAsync(dirPath).catch(
+ () => []
+ );
+
+ await Promise.all(
+ files.map(async (fileName) => {
+ const path = `${dirPath}/${fileName}`;
+ const info = await FileSystem.getInfoAsync(path).catch(() => null);
+ if (!info?.exists) return;
+
+ addFileToSnapshot(snapshot, {
+ path,
+ size: (info as any).size ?? 0,
+ modifiedAt: getFileModifiedAt(info),
+ ...parseCacheFile(dirName, fileName),
+ });
+ })
+ );
+ })
+ );
+
+ return snapshot;
+};
+
+const deleteCacheFiles = async (files: Iterable<CachedFile>) => {
+ let removed = 0;
+ await Promise.all(
+ Array.from(files).map(async (file) => {
+ removed += file.size;
+ await FileSystem.deleteAsync(file.path, { idempotent: true }).catch(
+ () => {}
+ );
+ })
+ );
+ return removed;
+};
+
+const cleanupUnexpectedFiles = async (
+ snapshot: CacheSnapshot,
+ expectedFormatsByLinkId: Map<number, Set<CacheFormat>>,
+ canDeleteOrphans: boolean
+) => {
+ const filesToDelete = new Map<string, CachedFile>();
+
+ for (const file of snapshot.partialFiles) {
+ filesToDelete.set(file.path, file);
+ }
+
+ for (const [linkId, files] of snapshot.filesByLinkId) {
+ const expectedFormats = expectedFormatsByLinkId.get(linkId);
+
+ if (!expectedFormats) {
+ if (canDeleteOrphans) {
+ for (const file of files) filesToDelete.set(file.path, file);
+ }
+ continue;
+ }
+
+ for (const file of files) {
+ if (file.format && !expectedFormats.has(file.format)) {
+ filesToDelete.set(file.path, file);
+ }
+ }
+ }
+
+ return deleteCacheFiles(filesToDelete.values());
+};
+
+const formatsForLink = (
+ link: LinkIncludingShortenedCollectionAndTags
+): CacheFormat[] => {
+ const formats: CacheFormat[] = [];
+
+ if (formatAvailable(link, "preview")) formats.push("preview");
+ if (formatAvailable(link, "readable")) formats.push("readable");
+ if (formatAvailable(link, "monolith")) formats.push("webpage");
+ if (formatAvailable(link, "pdf")) formats.push("pdf");
+ if (formatAvailable(link, "image")) {
+ const image =
+ typeof link.image === "string" ? link.image.toLowerCase() : "";
+ if (image.split("?")[0].endsWith("png")) {
+ formats.push("png");
+ } else {
+ formats.push("jpeg");
+ }
+ }
+
+ return formats;
+};
+
+const missingFormatsForLink = (
+ link: LinkIncludingShortenedCollectionAndTags & { id: number },
+ snapshot: CacheSnapshot
+): CacheFormat[] => {
+ const formats = formatsForLink(link);
+ const cachedFormats = snapshot.formatsByLinkId.get(link.id);
+ const linkFreshnessTime = getLinkFreshnessTime(link);
+
+ return formats.filter((format) => {
+ const file = cachedFormats?.get(format);
+ if (!file) return true;
+
+ return Boolean(
+ linkFreshnessTime &&
+ file.modifiedAt &&
+ file.modifiedAt + staleFileGraceMs < linkFreshnessTime
+ );
+ });
+};
+
+const downloadLinkFormats = async (
+ link: LinkIncludingShortenedCollectionAndTags & { id: number },
+ formats: CacheFormat[],
+ auth: MobileAuth,
+ userContentDomain: string | null | undefined
+) => {
+ const concurrency = 2;
+
+ for (let i = 0; i < formats.length; i += concurrency) {
+ if (cancelled) return;
+
+ const slice = formats.slice(i, i + concurrency);
+ await Promise.all(
+ slice.map(async (format) => {
+ try {
+ const { delta } = await fetchFormatToCache({
+ link: { id: link.id, updatedAt: link.updatedAt },
+ format,
+ auth,
+ userContentDomain,
+ });
+ useOfflineSyncStore.getState().addBytes(delta);
+ } catch (e) {
+ console.warn(
+ `[offlineSync] Failed to cache link ${link.id} format ${format}`,
+ e
+ );
+ useOfflineSyncStore.getState().incrementFailed();
+ }
+ })
+ );
+ }
+};
+
+export const startSync = async (
+ auth: MobileAuth,
+ userContentDomain: string | null | undefined
+) => {
+ const generation = ++syncGeneration;
+ activeAuth = auth;
+ activeUserContentDomain = userContentDomain;
+
+ if (runPromise) {
+ cancelled = true;
+ await runPromise;
+ if (generation !== syncGeneration) return;
+ }
+
+ const net = await NetInfo.fetch();
+ if (generation !== syncGeneration) return;
+ if (!isOnline(net)) {
+ useOfflineSyncStore.getState().setStatus("paused");
+ return;
+ }
+
+ cancelled = false;
+
+ const store = useOfflineSyncStore.getState();
+ store.setStatus("syncing");
+
+ const currentRun = (async () => {
+ try {
+ const { links, hasLinkSource } = getLinksFromCache();
+ const cacheSnapshot = await scanCacheSnapshot();
+ const expectedFormatsByLinkId = new Map<number, Set<CacheFormat>>();
+
+ for (const link of links) {
+ if (typeof link.id !== "number") continue;
+ expectedFormatsByLinkId.set(link.id, new Set(formatsForLink(link)));
+ }
+
+ const removedBytes = await cleanupUnexpectedFiles(
+ cacheSnapshot,
+ expectedFormatsByLinkId,
+ hasLinkSource
+ );
+ store.setBytesUsed(Math.max(0, cacheSnapshot.bytesUsed - removedBytes));
+
+ store.setProgress(0, links.length);
+
+ for (const link of links) {
+ if (cancelled || generation !== syncGeneration) break;
+ if (link.id === undefined) continue;
+
+ const fullLink = link as LinkIncludingShortenedCollectionAndTags & {
+ id: number;
+ };
+
+ setCachedLinkQueryData(fullLink);
+
+ const missing = missingFormatsForLink(fullLink, cacheSnapshot);
+ if (missing.length === 0) {
+ store.incrementProcessed();
+ continue;
+ }
+
+ store.setCurrentLinkId(fullLink.id);
+ await downloadLinkFormats(fullLink, missing, auth, userContentDomain);
+ store.incrementProcessed();
+ }
+ } catch (e) {
+ console.error("[offlineSync] Sync failed", e);
+ } finally {
+ const currentStore = useOfflineSyncStore.getState();
+ currentStore.setCurrentLinkId(null);
+ if (generation === syncGeneration && currentStore.status !== "paused") {
+ currentStore.setStatus("idle");
+ }
+ }
+ })();
+
+ runPromise = currentRun;
+
+ try {
+ await currentRun;
+ } finally {
+ if (runPromise === currentRun) runPromise = null;
+ }
+};
+
+export const stopSync = () => {
+ syncGeneration += 1;
+ cancelled = true;
+ activeAuth = null;
+ activeUserContentDomain = undefined;
+ const store = useOfflineSyncStore.getState();
+ store.setCurrentLinkId(null);
+ if (store.status !== "idle") store.setStatus("idle");
+};
+
+export const recomputeStorage = async () => {
+ const bytes = await getCacheSize();
+ useOfflineSyncStore.getState().setBytesUsed(bytes);
+};
+
+export const subscribeToConnectivity = () => {
+ if (netUnsubscribe) return;
+
+ netUnsubscribe = NetInfo.addEventListener((state) => {
+ const store = useOfflineSyncStore.getState();
+ if (!isOnline(state)) {
+ if (runPromise) {
+ cancelled = true;
+ store.setStatus("paused");
+ }
+ } else {
+ if (store.status === "paused" && activeAuth && !runPromise) {
+ startSync(activeAuth, activeUserContentDomain);
+ }
+ }
+ });
+};
+
+export const subscribeToQueryCache = () => {
+ if (cacheUnsubscribe) return;
+
+ cacheUnsubscribe = queryClient.getQueryCache().subscribe((event) => {
+ if (syncQueryWriteDepth > 0) return;
+
+ const key = event.query.queryKey[0];
+ if (key !== "links" && key !== "dashboardData" && key !== "link") return;
+ if (!activeAuth) return;
+
+ if (cacheSyncTimeout) clearTimeout(cacheSyncTimeout);
+ cacheSyncTimeout = setTimeout(() => {
+ if (activeAuth) startSync(activeAuth, activeUserContentDomain);
+ }, 1500);
+ });
+};
+
+export const unsubscribeFromQueryCache = () => {
+ cacheUnsubscribe?.();
+ cacheUnsubscribe = null;
+ if (cacheSyncTimeout) clearTimeout(cacheSyncTimeout);
+ cacheSyncTimeout = null;
+};
+
+export const unsubscribeFromConnectivity = () => {
+ netUnsubscribe?.();
+ netUnsubscribe = null;
+};
+
+export const formatBytes = (bytes: number): string => {
+ if (bytes < 1024) return `${bytes} B`;
+ if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+ if (bytes < 1024 * 1024 * 1024)
+ return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+ return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
+};
diff --git a/apps/mobile/lib/queryPersister.ts b/apps/mobile/lib/queryPersister.ts
index 4e2a7beb..d7ec3fcf 100644
--- a/apps/mobile/lib/queryPersister.ts
+++ b/apps/mobile/lib/queryPersister.ts
@@ -1,8 +1,46 @@
import { createMMKV } from "react-native-mmkv";
-import { Persister } from "@tanstack/react-query-persist-client";
+import type {
+ PersistedClient,
+ Persister,
+} from "@tanstack/react-query-persist-client";
const storage = createMMKV({ id: "react-query" });
+const infiniteQueryKeys = new Set(["links", "publicLinks", "tags"]);
+
+const isInfiniteQueryKey = (queryKey: unknown) =>
+ Array.isArray(queryKey) &&
+ typeof queryKey[0] === "string" &&
+ infiniteQueryKeys.has(queryKey[0]);
+
+const hasInfiniteDataShape = (data: unknown) =>
+ data == null ||
+ (typeof data === "object" &&
+ Array.isArray((data as any).pages) &&
+ Array.isArray((data as any).pageParams));
+
+export const sanitizePersistedClient = (
+ client: PersistedClient
+): PersistedClient => {
+ const queries = client.clientState?.queries;
+ if (!Array.isArray(queries)) return client;
+
+ const sanitizedQueries = queries.filter((query) => {
+ if (!isInfiniteQueryKey(query.queryKey)) return true;
+ return hasInfiniteDataShape(query.state?.data);
+ });
+
+ if (sanitizedQueries.length === queries.length) return client;
+
+ return {
+ ...client,
+ clientState: {
+ ...client.clientState,
+ queries: sanitizedQueries,
+ },
+ };
+};
+
export const mmkvPersister: Persister = {
persistClient: async (client) => {
try {
@@ -15,7 +53,7 @@ export const mmkvPersister: Persister = {
restoreClient: async () => {
try {
const json = storage.getString("REACT_QUERY_CACHE");
- return json ? JSON.parse(json) : undefined;
+ return json ? sanitizePersistedClient(JSON.parse(json)) : undefined;
} catch (e) {
console.error("Error restoring client:", e);
return undefined;
diff --git a/apps/mobile/store/auth.ts b/apps/mobile/store/auth.ts
index f54f44d8..2e31ef71 100644
--- a/apps/mobile/store/auth.ts
+++ b/apps/mobile/store/auth.ts
@@ -6,6 +6,8 @@ import { Alert } from "react-native";
import { queryClient } from "@/lib/queryClient";
import { mmkvPersister } from "@/lib/queryPersister";
import { clearCache } from "@/lib/cache";
+import useDataStore from "@/store/data";
+import { useOfflineSyncStore } from "@/lib/offlineSync";
type AuthStore = {
auth: MobileAuth;
@@ -142,6 +144,9 @@ const useAuthStore = create<AuthStore>((set) => ({
await clearCache();
+ useDataStore.getState().updateData({ offlineEnabled: false });
+ useOfflineSyncStore.getState().reset();
+
set({
auth: {
instance: "",
diff --git a/apps/mobile/store/data.ts b/apps/mobile/store/data.ts
index 3f0a0549..56660222 100644
--- a/apps/mobile/store/data.ts
+++ b/apps/mobile/store/data.ts
@@ -18,6 +18,7 @@ const useDataStore = create<DataStore>((set, get) => ({
theme: "system",
preferredBrowser: "app",
preferredCollection: null,
+ offlineEnabled: false,
},
setData: async () => {
const dataString = JSON.parse((await AsyncStorage.getItem("data")) || "{}");
diff --git a/apps/web/components/SortDropdown.tsx b/apps/web/components/SortDropdown.tsx
index 58f49849..f5171388 100644
--- a/apps/web/components/SortDropdown.tsx
+++ b/apps/web/components/SortDropdown.tsx
@@ -2,7 +2,7 @@ import React, { Dispatch, SetStateAction, useEffect } from "react";
import { Sort } from "@linkwarden/types/global";
import { TFunction } from "i18next";
import useLocalSettingsStore from "@/store/localSettings";
-import { resetInfiniteQueryPagination } from "@linkwarden/router/links";
+import { resetInfiniteQueryPagination } from "@linkwarden/router/lib";
import { useQueryClient } from "@tanstack/react-query";
import {
DropdownMenu,
diff --git a/packages/router/config.tsx b/packages/router/config.tsx
index f949ed4b..334c80d2 100644
--- a/packages/router/config.tsx
+++ b/packages/router/config.tsx
@@ -49,8 +49,12 @@ const isAtLeastInstanceVersion = (
};
const useConfig = (auth?: MobileAuth) => {
+ const isMobile = auth !== undefined;
+
return useQuery({
queryKey: ["config", auth?.instance ?? "web"],
+ enabled:
+ !isMobile || (auth.status === "authenticated" && Boolean(auth.instance)),
queryFn: async () => {
const response = await fetch(
(auth?.instance ? auth.instance : "") + "/api/v1/config",
diff --git a/packages/router/lib.ts b/packages/router/lib.ts
new file mode 100644
index 00000000..5a4b4c48
--- /dev/null
+++ b/packages/router/lib.ts
@@ -0,0 +1,18 @@
+import { QueryClient, QueryKey } from "@tanstack/react-query";
+
+const resetInfiniteQueryPagination = async (
+ queryClient: QueryClient,
+ queryKey: QueryKey
+) => {
+ await queryClient.cancelQueries({ queryKey });
+ queryClient.setQueriesData({ queryKey }, (oldData: any) => {
+ if (!oldData) return undefined;
+ return {
+ pages: oldData.pages.slice(0, 1),
+ pageParams: oldData.pageParams.slice(0, 1),
+ };
+ });
+ await queryClient.invalidateQueries({ queryKey });
+};
+
+export { resetInfiniteQueryPagination };
diff --git a/packages/router/links.tsx b/packages/router/links.tsx
index e797b298..1ee33347 100644
--- a/packages/router/links.tsx
+++ b/packages/router/links.tsx
@@ -1121,22 +1121,6 @@ const useArchiveAction = () => {
});
};
-const resetInfiniteQueryPagination = async (
- queryClient: any,
- queryKey: any
-) => {
- queryClient.setQueriesData({ queryKey }, (oldData: any) => {
- if (!oldData) return undefined;
-
- return {
- pages: oldData.pages.slice(0, 1),
- pageParams: oldData.pageParams.slice(0, 1),
- };
- });
-
- await queryClient.invalidateQueries(queryKey);
-};
-
export {
useLinks,
useAddLink,
@@ -1147,6 +1131,5 @@ export {
useGetLink,
useBulkEditLinks,
useArchiveAction,
- resetInfiniteQueryPagination,
useUpdateFile,
};
diff --git a/packages/types/global.ts b/packages/types/global.ts
index 59136e44..a09db28d 100644
--- a/packages/types/global.ts
+++ b/packages/types/global.ts
@@ -209,6 +209,7 @@ export interface MobileData {
theme: "light" | "dark" | "system";
preferredBrowser: "app" | "system";
preferredCollection: CollectionIncludingMembersAndLinkCount | null;
+ offlineEnabled: boolean;
}
export interface WorkerStats {