Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

refactor(mobile): replace react-mmkv with expo-sqlite

daniel31x1387ec3d3be3a8
diff --git a/apps/mobile/app/(tabs)/_layout.tsx b/apps/mobile/app/(tabs)/_layout.tsx
index b5614ed9..ef9e7921 100644
--- a/apps/mobile/app/(tabs)/_layout.tsx
+++ b/apps/mobile/app/(tabs)/_layout.tsx
@@ -14,11 +14,9 @@ import useAuthStore from "@/store/auth";
import { useUser } from "@linkwarden/router/user";
import { useConfig } from "@linkwarden/router/config";
import { shouldRouteToSubscribe } from "@/lib/subscription";
-import { queryClient } from "@/lib/queryClient";
-const isCacheEmpty = (instance: string) =>
- queryClient.getQueryData(["user"]) == null &&
- queryClient.getQueryData(["config", instance]) == null;
+const isRejectedSession = (error: unknown) =>
+ (error as any)?.status === 401 || (error as any)?.status === 403;
let recoveringSession: string | null = null;
@@ -28,28 +26,20 @@ export default function TabLayout() {
const { auth } = useAuthStore();
const {
data: user,
- isError: isUserError,
+ error: userError,
isLoading: isUserLoading,
} = useUser(auth);
const config = useConfig(auth);
const routeToSubscribe = shouldRouteToSubscribe(user, config.data);
- const hasEmptyCache = auth.instance ? isCacheEmpty(auth.instance) : false;
const shouldRecover =
- auth.status === "authenticated" &&
- isUserError &&
- config.isError &&
- hasEmptyCache;
+ auth.status === "authenticated" && isRejectedSession(userError);
useEffect(() => {
if (routeToSubscribe) router.replace("/subscribe");
}, [routeToSubscribe, router]);
useEffect(() => {
- if (
- !shouldRecover ||
- !auth.session ||
- recoveringSession === auth.session
- )
+ if (!shouldRecover || !auth.session || recoveringSession === auth.session)
return;
recoveringSession = auth.session;
diff --git a/apps/mobile/app/(tabs)/dashboard/index.tsx b/apps/mobile/app/(tabs)/dashboard/index.tsx
index f46f7eb0..a73c1523 100644
--- a/apps/mobile/app/(tabs)/dashboard/index.tsx
+++ b/apps/mobile/app/(tabs)/dashboard/index.tsx
@@ -43,12 +43,14 @@ export default function DashboardScreen() {
const { data: user, ...userData } = useUser(auth);
const { data: collections = [], ...collectionsData } = useCollections(auth);
const config = useConfig(auth);
+ const instanceVersion = config.data?.INSTANCE_VERSION;
const supportsDashboardTagCount = isAtLeastInstanceVersion(
- config.data?.INSTANCE_VERSION,
+ instanceVersion,
DASHBOARD_TAG_COUNT_VERSION
);
- const shouldUseLegacyTagsCount =
- config.isError || (config.isSuccess && !supportsDashboardTagCount);
+ const shouldUseLegacyTagsCount = instanceVersion
+ ? !supportsDashboardTagCount
+ : config.isError;
const legacyTags = useTags(auth, {
enabled: shouldUseLegacyTagsCount,
});
@@ -84,9 +86,10 @@ export default function DashboardScreen() {
}, [dashboardSections]);
const [pullRefreshing, setPullRefreshing] = useState(false);
- const resolvedNumberOfTags = shouldUseLegacyTagsCount
- ? legacyTags.data.length
- : numberOfTags;
+ const resolvedNumberOfTags =
+ shouldUseLegacyTagsCount && legacyTags.isSuccess
+ ? legacyTags.data.length
+ : numberOfTags;
const onRefresh = async () => {
setPullRefreshing(true);
diff --git a/apps/mobile/app/(tabs)/settings/index.tsx b/apps/mobile/app/(tabs)/settings/index.tsx
index 66806883..a65b473e 100644
--- a/apps/mobile/app/(tabs)/settings/index.tsx
+++ b/apps/mobile/app/(tabs)/settings/index.tsx
@@ -71,10 +71,10 @@ export default function SettingsScreen() {
const syncStatusLabel =
syncStatus === "paused"
? "Waiting for connection"
- : syncPercent === null
- ? "Preparing…"
- : syncPercent >= 100
- ? "Up to date"
+ : syncStatus !== "syncing"
+ ? "Up to date"
+ : syncPercent === null
+ ? "Preparing…"
: `${syncPercent}%`;
return (
diff --git a/apps/mobile/app/_layout.tsx b/apps/mobile/app/_layout.tsx
index 9e70ae02..bd84c192 100644
--- a/apps/mobile/app/_layout.tsx
+++ b/apps/mobile/app/_layout.tsx
@@ -5,7 +5,7 @@ import {
useRouter,
} from "expo-router";
import { PersistQueryClientProvider } from "@tanstack/react-query-persist-client";
-import { mmkvPersister } from "@/lib/queryPersister";
+import { queryPersister } from "@/lib/queryPersister";
import { useState, useEffect } from "react";
import "../styles/global.css";
import { SheetProvider } from "react-native-actions-sheet";
@@ -90,11 +90,11 @@ export default Sentry.wrap(function RootLayout() {
<PersistQueryClientProvider
client={queryClient}
persistOptions={{
- persister: mmkvPersister,
+ persister: queryPersister,
maxAge: Infinity,
dehydrateOptions: {
shouldDehydrateMutation: () => true,
- shouldDehydrateQuery: (query) => query.state.status === "success",
+ shouldDehydrateQuery: (query) => query.state.data !== undefined,
},
}}
onSuccess={() => {
diff --git a/apps/mobile/app/index.tsx b/apps/mobile/app/index.tsx
index 4e3998a7..aaf6033f 100644
--- a/apps/mobile/app/index.tsx
+++ b/apps/mobile/app/index.tsx
@@ -120,7 +120,7 @@ export default function HomeScreen() {
.then(() => {
Alert.alert(
"Signed out",
- "The server could not be reached and no cached user data was available. Check your connection and sign in again."
+ "Your session is no longer valid. Please sign in again."
);
})
.catch((error) => {
diff --git a/apps/mobile/components/LinkListing.tsx b/apps/mobile/components/LinkListing.tsx
index 592d2fb7..e044a1cf 100644
--- a/apps/mobile/components/LinkListing.tsx
+++ b/apps/mobile/components/LinkListing.tsx
@@ -175,9 +175,8 @@ const LinkListing = ({ link, dashboard }: Props) => {
{formatAvailable(link, "preview") ? (
preview ? (
<Image
- source={{
- uri: `${preview}?updatedAt=${link.updatedAt}`,
- }}
+ key={String(link.updatedAt)}
+ source={{ uri: preview }}
className="rounded-md h-[60px] w-[90px] object-cover scale-105"
/>
) : (
diff --git a/apps/mobile/lib/cache.ts b/apps/mobile/lib/cache.ts
index f6418183..b104d247 100644
--- a/apps/mobile/lib/cache.ts
+++ b/apps/mobile/lib/cache.ts
@@ -3,6 +3,7 @@ import NetInfo from "@react-native-community/netinfo";
import { ArchivedFormat, MobileAuth } from "@linkwarden/types/global";
import getPreservedFormatUrl from "@linkwarden/lib/getPreservedFormatUrl";
import { customHeadersFor } from "@/lib/customHeaders";
+import { queryPersister } from "@/lib/queryPersister";
type LoadCacheOrFetchOptions<T> = {
filePath: string;
@@ -141,6 +142,121 @@ type FetchFormatToCacheOptions = {
format: CacheFormat;
auth: MobileAuth;
userContentDomain?: string | null;
+ onProgress?: (deltaBytes: number) => void;
+};
+
+const connectionErrorFlag = "isConnectionError";
+
+const connectionError = (cause: unknown) =>
+ Object.assign(
+ new Error(
+ `Could not reach the server: ${(cause as any)?.message ?? cause}`
+ ),
+ { [connectionErrorFlag]: true }
+ );
+
+export const isConnectionError = (e: unknown): boolean =>
+ Boolean(e && typeof e === "object" && (e as any)[connectionErrorFlag]);
+
+const unavailableStatuses = new Set([502, 503, 504]);
+
+const failedResponse = (status: number) => {
+ const error = new Error(`HTTP ${status}`);
+ return unavailableStatuses.has(status)
+ ? Object.assign(error, { [connectionErrorFlag]: true })
+ : error;
+};
+
+const cancelledFlag = "isCancelledTransfer";
+
+const cancelledError = () =>
+ Object.assign(new Error("Transfer cancelled"), { [cancelledFlag]: true });
+
+export const isCancelledTransfer = (e: unknown): boolean =>
+ Boolean(e && typeof e === "object" && (e as any)[cancelledFlag]);
+
+const stallTimeoutMs = 30_000;
+
+const progressStepBytes = Math.round((1024 * 1024) / 10);
+
+const activeTransfers = new Set<{ cancel: () => void }>();
+
+export const cancelActiveTransfers = () => {
+ for (const transfer of [...activeTransfers]) transfer.cancel();
+};
+
+const downloadWithStallTimeout = async ({
+ url,
+ toPath,
+ headers,
+ onBytesWritten,
+}: {
+ url: string;
+ toPath: string;
+ headers?: Record<string, string>;
+ onBytesWritten?: (totalBytesWritten: number) => void;
+}) => {
+ let stallTimer: ReturnType<typeof setTimeout> | null = null;
+ let interrupt: (e: Error) => void = () => {};
+ let settled = false;
+
+ const interrupted = new Promise<never>((_, reject) => {
+ interrupt = reject;
+ });
+
+ let lastSize = -1;
+ const checkStalled = async () => {
+ const info = await FileSystem.getInfoAsync(toPath).catch(() => null);
+ if (settled) return;
+
+ const size = info?.exists ? (info as any).size ?? 0 : 0;
+ if (size > lastSize) {
+ lastSize = size;
+ armStallTimer();
+ return;
+ }
+
+ interrupt(new Error("Download stalled"));
+ };
+
+ const armStallTimer = () => {
+ if (stallTimer) clearTimeout(stallTimer);
+ stallTimer = setTimeout(checkStalled, stallTimeoutMs);
+ };
+
+ const resumable = FileSystem.createDownloadResumable(
+ url,
+ toPath,
+ { headers },
+ ({ totalBytesWritten }) => {
+ if (settled) return;
+
+ onBytesWritten?.(totalBytesWritten);
+ armStallTimer();
+ }
+ );
+
+ const transfer = { cancel: () => interrupt(cancelledError()) };
+ activeTransfers.add(transfer);
+ armStallTimer();
+
+ const download = resumable.downloadAsync().catch((e) => {
+ throw connectionError(e);
+ });
+ download.catch(() => {});
+
+ try {
+ const result = await Promise.race([download, interrupted]);
+ if (!result) throw cancelledError();
+ return result;
+ } catch (e) {
+ resumable.cancelAsync().catch(() => {});
+ throw e;
+ } finally {
+ settled = true;
+ activeTransfers.delete(transfer);
+ if (stallTimer) clearTimeout(stallTimer);
+ }
};
export const fetchFormatToCache = async ({
@@ -148,6 +264,7 @@ export const fetchFormatToCache = async ({
format,
auth,
userContentDomain,
+ onProgress,
}: FetchFormatToCacheOptions): Promise<{
uri: string;
size: number;
@@ -185,12 +302,16 @@ export const fetchFormatToCache = async ({
};
if (format === "webpage" && userContentDomain) {
- apiUrl = await getPreservedFormatUrl({
- tokenEndpoint: `${auth.instance}/api/v1/preserved/token`,
- linkId: link.id,
- format: archivedFormat,
- headers,
- });
+ try {
+ apiUrl = await getPreservedFormatUrl({
+ tokenEndpoint: `${auth.instance}/api/v1/preserved/token`,
+ linkId: link.id,
+ format: archivedFormat,
+ headers,
+ });
+ } catch (e) {
+ throw connectionError(e);
+ }
headers = undefined;
} else {
apiUrl = `${auth.instance}/api/v1/archives/${link.id}?format=${archivedFormat}${previewSuffix}`;
@@ -199,21 +320,47 @@ export const fetchFormatToCache = async ({
const tmpPath = `${filePath}.part`;
await FileSystem.deleteAsync(tmpPath, { idempotent: true }).catch(() => {});
+ let reportedBytes = 0;
+ const reportBytesWritten = (totalBytesWritten: number) => {
+ if (!onProgress) return;
+
+ const target =
+ Math.floor(totalBytesWritten / progressStepBytes) * progressStepBytes;
+ if (target > reportedBytes) {
+ onProgress(target - reportedBytes);
+ reportedBytes = target;
+ }
+ };
+
try {
if (format === "readable") {
- const response = await fetch(apiUrl, { headers });
- if (!response.ok) throw new Error(`HTTP ${response.status}`);
- const data = (await response.json()).content;
+ const result = await downloadWithStallTimeout({
+ url: apiUrl,
+ toPath: tmpPath,
+ headers,
+ onBytesWritten: reportBytesWritten,
+ });
+ if (result.status < 200 || result.status >= 300) {
+ throw failedResponse(result.status);
+ }
+
+ const payload = await FileSystem.readAsStringAsync(tmpPath, {
+ encoding: FileSystem.EncodingType.UTF8,
+ });
+ const data = JSON.parse(payload).content;
await FileSystem.writeAsStringAsync(tmpPath, data, {
encoding: FileSystem.EncodingType.UTF8,
});
} else {
- const result = await FileSystem.downloadAsync(apiUrl, tmpPath, {
+ const result = await downloadWithStallTimeout({
+ url: apiUrl,
+ toPath: tmpPath,
headers: { ...customHeadersFor(apiUrl), ...headers },
+ onBytesWritten: reportBytesWritten,
});
if (result.status < 200 || result.status >= 300) {
- throw new Error(`HTTP ${result.status}`);
+ throw failedResponse(result.status);
}
}
@@ -245,9 +392,7 @@ export const clearCache = async () => {
FileSystem.deleteAsync(FileSystem.documentDirectory + "archivedData", {
idempotent: true,
}),
- FileSystem.deleteAsync(FileSystem.documentDirectory + "mmkv", {
- idempotent: true,
- }),
+ queryPersister.removeClient?.(),
]);
};
diff --git a/apps/mobile/lib/offlineSync.ts b/apps/mobile/lib/offlineSync.ts
index 91f485ed..0f7af852 100644
--- a/apps/mobile/lib/offlineSync.ts
+++ b/apps/mobile/lib/offlineSync.ts
@@ -1,12 +1,20 @@
import { create } from "zustand";
import NetInfo from "@react-native-community/netinfo";
+import { notifyManager } from "@tanstack/react-query";
import {
LinkIncludingShortenedCollectionAndTags,
MobileAuth,
} from "@linkwarden/types/global";
import { formatAvailable } from "@linkwarden/lib/formatStats";
import * as FileSystem from "expo-file-system/legacy";
-import { CacheFormat, fetchFormatToCache, getCacheSize } from "@/lib/cache";
+import {
+ CacheFormat,
+ cancelActiveTransfers,
+ fetchFormatToCache,
+ getCacheSize,
+ isCancelledTransfer,
+ isConnectionError,
+} from "@/lib/cache";
import { queryClient } from "@/lib/queryClient";
type SyncStatus = "idle" | "syncing" | "paused";
@@ -63,7 +71,8 @@ let cacheSyncTimeout: ReturnType<typeof setTimeout> | null = null;
let activeAuth: MobileAuth | null = null;
let activeUserContentDomain: string | null | undefined = undefined;
let syncQueryWriteDepth = 0;
-let ignoredQueryKeyClearTimeout: ReturnType<typeof setTimeout> | null = null;
+let pendingRescan = false;
+let storageSettleTimeout: ReturnType<typeof setTimeout> | null = null;
const archiveRoot = FileSystem.documentDirectory + "archivedData";
const staleFileGraceMs = 5000;
@@ -122,26 +131,31 @@ const isOnline = (state: {
isInternetReachable?: boolean | null;
}) => state.isConnected === true && state.isInternetReachable !== false;
-const ignoredQueryKeySignatures = new Set<string>();
+const connectionRetryMs = 30_000;
-const getQueryKeySignature = (queryKey: unknown) => {
- try {
- return JSON.stringify(queryKey);
- } catch {
- return undefined;
- }
+let connectionRetryTimeout: ReturnType<typeof setTimeout> | null = null;
+
+const clearConnectionRetry = () => {
+ if (!connectionRetryTimeout) return;
+ clearTimeout(connectionRetryTimeout);
+ connectionRetryTimeout = null;
};
-const ignoreQueryKeyTemporarily = (queryKey: unknown) => {
- const signature = getQueryKeySignature(queryKey);
- if (!signature) return;
+const scheduleConnectionRetry = () => {
+ if (connectionRetryTimeout) return;
- ignoredQueryKeySignatures.add(signature);
- if (ignoredQueryKeyClearTimeout) clearTimeout(ignoredQueryKeyClearTimeout);
- ignoredQueryKeyClearTimeout = setTimeout(() => {
- ignoredQueryKeySignatures.clear();
- ignoredQueryKeyClearTimeout = null;
- }, 500);
+ connectionRetryTimeout = setTimeout(() => {
+ connectionRetryTimeout = null;
+ if (activeAuth) startSync(activeAuth, activeUserContentDomain);
+ }, connectionRetryMs);
+};
+
+let connectionPauses = 0;
+
+const pauseForConnection = () => {
+ connectionPauses += 1;
+ useOfflineSyncStore.getState().setStatus("paused");
+ scheduleConnectionRetry();
};
const sameArchiveState = (a: any, b: LinkIncludingShortenedCollectionAndTags) =>
@@ -160,21 +174,36 @@ const setCachedLinkQueryData = (
const existing = queryClient.getQueryData(queryKey);
if (sameArchiveState(existing, link)) return;
- ignoreQueryKeyTemporarily(queryKey);
- syncQueryWriteDepth += 1;
- try {
- queryClient.setQueryData(queryKey, link);
- } finally {
- syncQueryWriteDepth -= 1;
- }
+ queryClient.setQueryData(queryKey, link);
+};
+
+const hydrateLinkQueries = (
+ links: LinkIncludingShortenedCollectionAndTags[]
+) => {
+ notifyManager.batch(() => {
+ syncQueryWriteDepth += 1;
+ try {
+ for (const link of links) {
+ if (typeof link.id !== "number") continue;
+ setCachedLinkQueryData(
+ link as LinkIncludingShortenedCollectionAndTags & { id: number }
+ );
+ }
+ } finally {
+ syncQueryWriteDepth -= 1;
+ }
+ });
};
const getLinksFromCache = (): LinksCacheSnapshot => {
const linksById = new Map<number, LinkIncludingShortenedCollectionAndTags>();
- const add = (link: any) => {
+ const dashboardLinkIds = new Set<number>();
+ const add = (link: any, fromDashboard = false) => {
const id = Number(link?.id);
if (!Number.isFinite(id)) return;
+ if (fromDashboard) dashboardLinkIds.add(id);
+
const existing = linksById.get(id);
if (
!existing ||
@@ -197,16 +226,34 @@ const getLinksFromCache = (): LinksCacheSnapshot => {
for (const link of page?.links ?? []) add(link);
}
} else if (key[0] === "dashboardData") {
- if (Array.isArray(data.links)) hasLinkSource = true;
- for (const link of data.links ?? []) add(link);
+ const collectionLinks =
+ data.collectionLinks && typeof data.collectionLinks === "object"
+ ? Object.values(
+ data.collectionLinks as Record<string, unknown>
+ ).flatMap((links) => (Array.isArray(links) ? links : []))
+ : [];
+
+ if (Array.isArray(data.links) || collectionLinks.length > 0) {
+ hasLinkSource = true;
+ }
+
+ for (const link of data.links ?? []) add(link, true);
+ for (const link of collectionLinks) add(link, true);
} else if (key[0] === "link") {
if (Number.isFinite(Number(data?.id))) hasLinkSource = true;
add(data);
}
}
+ const links = Array.from(linksById.values());
+ const isDashboardLink = (link: LinkIncludingShortenedCollectionAndTags) =>
+ dashboardLinkIds.has(Number(link.id));
+
return {
- links: Array.from(linksById.values()),
+ links: [
+ ...links.filter(isDashboardLink),
+ ...links.filter((link) => !isDashboardLink(link)),
+ ],
hasLinkSource,
};
};
@@ -394,22 +441,40 @@ const downloadLinkFormats = async (
userContentDomain: string | null | undefined
) => {
const concurrency = 2;
+ let lostConnection = false;
for (let i = 0; i < formats.length; i += concurrency) {
- if (cancelled) return;
+ if (cancelled || lostConnection) break;
const slice = formats.slice(i, i + concurrency);
await Promise.all(
slice.map(async (format) => {
+ let streamedBytes = 0;
+
try {
const { delta } = await fetchFormatToCache({
link: { id: link.id, updatedAt: link.updatedAt },
format,
auth,
userContentDomain,
+ onProgress: (deltaBytes) => {
+ streamedBytes += deltaBytes;
+ useOfflineSyncStore.getState().addBytes(deltaBytes);
+ },
});
- useOfflineSyncStore.getState().addBytes(delta);
+ useOfflineSyncStore.getState().addBytes(delta - streamedBytes);
} catch (e) {
+ if (streamedBytes) {
+ useOfflineSyncStore.getState().addBytes(-streamedBytes);
+ }
+
+ if (isCancelledTransfer(e)) return;
+
+ if (isConnectionError(e)) {
+ lostConnection = true;
+ return;
+ }
+
console.warn(
`[offlineSync] Failed to cache link ${link.id} format ${format}`,
e
@@ -419,18 +484,43 @@ const downloadLinkFormats = async (
})
);
}
+
+ return { lostConnection };
+};
+
+const sameSyncTarget = (
+ auth: MobileAuth,
+ userContentDomain: string | null | undefined
+) =>
+ activeAuth?.session === auth.session &&
+ activeAuth?.instance === auth.instance &&
+ activeUserContentDomain === userContentDomain;
+
+const sameIdSet = (a: Set<number>, b: Set<number>) => {
+ if (a.size !== b.size) return false;
+ for (const id of a) if (!b.has(id)) return false;
+ return true;
};
export const startSync = async (
auth: MobileAuth,
userContentDomain: string | null | undefined
) => {
- const generation = ++syncGeneration;
+ const targetUnchanged = sameSyncTarget(auth, userContentDomain);
+
activeAuth = auth;
activeUserContentDomain = userContentDomain;
+ if (runPromise && targetUnchanged) {
+ pendingRescan = true;
+ return runPromise;
+ }
+
+ const generation = ++syncGeneration;
+
if (runPromise) {
cancelled = true;
+ cancelActiveTransfers();
await runPromise;
if (generation !== syncGeneration) return;
}
@@ -438,61 +528,122 @@ export const startSync = async (
const net = await NetInfo.fetch();
if (generation !== syncGeneration) return;
if (!isOnline(net)) {
- useOfflineSyncStore.getState().setStatus("paused");
+ pauseForConnection();
return;
}
cancelled = false;
+ clearConnectionRetry();
const store = useOfflineSyncStore.getState();
- store.setStatus("syncing");
+ let waitingForConnection = store.status === "paused";
const currentRun = (async () => {
+ const pausesAtStart = connectionPauses;
+ const processedRevisions = new Map<number, unknown>();
+ let previousLinkIds: Set<number> | null = null;
+
try {
- const { links, hasLinkSource } = getLinksFromCache();
- const cacheSnapshot = await scanCacheSnapshot();
- const expectedFormatsByLinkId = new Map<number, Set<CacheFormat>>();
+ do {
+ pendingRescan = false;
- for (const link of links) {
- if (typeof link.id !== "number") continue;
- expectedFormatsByLinkId.set(link.id, new Set(formatsForLink(link)));
- }
+ const { links, hasLinkSource } = getLinksFromCache();
- const removedBytes = await cleanupUnexpectedFiles(
- cacheSnapshot,
- expectedFormatsByLinkId,
- hasLinkSource
- );
- store.setBytesUsed(Math.max(0, cacheSnapshot.bytesUsed - removedBytes));
+ hydrateLinkQueries(links);
- store.setProgress(0, links.length);
+ const linkIds = new Set<number>();
+ const queue: (LinkIncludingShortenedCollectionAndTags & {
+ id: number;
+ })[] = [];
+
+ for (const link of links) {
+ if (typeof link.id !== "number") continue;
+ linkIds.add(link.id);
+
+ if (
+ !processedRevisions.has(link.id) ||
+ processedRevisions.get(link.id) !== (link as any).updatedAt
+ ) {
+ queue.push(
+ link as LinkIncludingShortenedCollectionAndTags & { id: number }
+ );
+ }
+ }
- for (const link of links) {
- if (cancelled || generation !== syncGeneration) break;
- if (link.id === undefined) continue;
+ const linkSetChanged =
+ !previousLinkIds || !sameIdSet(previousLinkIds, linkIds);
+ previousLinkIds = linkIds;
- const fullLink = link as LinkIncludingShortenedCollectionAndTags & {
- id: number;
- };
+ if (queue.length === 0 && !linkSetChanged) continue;
- setCachedLinkQueryData(fullLink);
+ const cacheSnapshot = await scanCacheSnapshot();
+ const expectedFormatsByLinkId = new Map<number, Set<CacheFormat>>();
- const missing = missingFormatsForLink(fullLink, cacheSnapshot);
- if (missing.length === 0) {
- store.incrementProcessed();
- continue;
+ for (const link of links) {
+ if (typeof link.id !== "number") continue;
+ expectedFormatsByLinkId.set(link.id, new Set(formatsForLink(link)));
}
- store.setCurrentLinkId(fullLink.id);
- await downloadLinkFormats(fullLink, missing, auth, userContentDomain);
- store.incrementProcessed();
- }
+ const removedBytes = await cleanupUnexpectedFiles(
+ cacheSnapshot,
+ expectedFormatsByLinkId,
+ hasLinkSource
+ );
+ store.setBytesUsed(Math.max(0, cacheSnapshot.bytesUsed - removedBytes));
+
+ const work: {
+ link: LinkIncludingShortenedCollectionAndTags & { id: number };
+ missing: CacheFormat[];
+ }[] = [];
+
+ for (const fullLink of queue) {
+ const missing = missingFormatsForLink(fullLink, cacheSnapshot);
+ if (missing.length > 0) {
+ work.push({ link: fullLink, missing });
+ } else {
+ processedRevisions.set(fullLink.id, (fullLink as any).updatedAt);
+ }
+ }
+
+ if (work.length === 0) continue;
+
+ store.setProgress(linkIds.size - work.length, linkIds.size);
+ if (!waitingForConnection) store.setStatus("syncing");
+
+ for (const { link: fullLink, missing } of work) {
+ if (cancelled || generation !== syncGeneration) break;
+
+ store.setCurrentLinkId(fullLink.id);
+ const { lostConnection } = await downloadLinkFormats(
+ fullLink,
+ missing,
+ auth,
+ userContentDomain
+ );
+
+ if (lostConnection) {
+ if (generation === syncGeneration) {
+ cancelled = true;
+ pauseForConnection();
+ }
+ break;
+ }
+
+ if (waitingForConnection) {
+ waitingForConnection = false;
+ store.setStatus("syncing");
+ }
+
+ processedRevisions.set(fullLink.id, (fullLink as any).updatedAt);
+ store.incrementProcessed();
+ }
+ } while (pendingRescan && !cancelled && generation === syncGeneration);
} catch (e) {
console.error("[offlineSync] Sync failed", e);
} finally {
const currentStore = useOfflineSyncStore.getState();
currentStore.setCurrentLinkId(null);
- if (generation === syncGeneration && currentStore.status !== "paused") {
+ if (generation === syncGeneration && connectionPauses === pausesAtStart) {
currentStore.setStatus("idle");
}
}
@@ -510,11 +661,20 @@ export const startSync = async (
export const stopSync = () => {
syncGeneration += 1;
cancelled = true;
+ pendingRescan = false;
+ cancelActiveTransfers();
+ clearConnectionRetry();
activeAuth = null;
activeUserContentDomain = undefined;
const store = useOfflineSyncStore.getState();
store.setCurrentLinkId(null);
if (store.status !== "idle") store.setStatus("idle");
+
+ if (storageSettleTimeout) clearTimeout(storageSettleTimeout);
+ storageSettleTimeout = setTimeout(() => {
+ storageSettleTimeout = null;
+ if (!runPromise) recomputeStorage();
+ }, 3000);
};
export const recomputeStorage = async () => {
@@ -526,17 +686,24 @@ export const subscribeToConnectivity = () => {
if (netUnsubscribe) return;
netUnsubscribe = NetInfo.addEventListener((state) => {
- const store = useOfflineSyncStore.getState();
+ if (!activeAuth) return;
+
if (!isOnline(state)) {
- if (runPromise) {
- cancelled = true;
- store.setStatus("paused");
- }
- } else {
- if (store.status === "paused" && activeAuth && !runPromise) {
- startSync(activeAuth, activeUserContentDomain);
- }
+ cancelled = true;
+ cancelActiveTransfers();
+ pauseForConnection();
+ return;
}
+
+ if (useOfflineSyncStore.getState().status !== "paused") return;
+
+ if (runPromise) {
+ scheduleConnectionRetry();
+ return;
+ }
+
+ clearConnectionRetry();
+ startSync(activeAuth, activeUserContentDomain);
});
};
diff --git a/apps/mobile/lib/queryPersister.ts b/apps/mobile/lib/queryPersister.ts
index 4d2dfcbe..d8c3c159 100644
--- a/apps/mobile/lib/queryPersister.ts
+++ b/apps/mobile/lib/queryPersister.ts
@@ -1,19 +1,43 @@
-import { createMMKV } from "react-native-mmkv";
+import * as SQLite from "expo-sqlite";
+import * as FileSystem from "expo-file-system/legacy";
+import { AppState } from "react-native";
import type {
PersistedClient,
Persister,
} from "@tanstack/react-query-persist-client";
-const storage = createMMKV({ id: "react-query" });
+const DATABASE_NAME = "react-query.db";
const CACHE_KEY = "REACT_QUERY_CACHE";
-let persistenceBroken = false;
+const deleteLegacyStorage = () =>
+ FileSystem.deleteAsync(FileSystem.documentDirectory + "mmkv", {
+ idempotent: true,
+ }).catch(() => {});
-const shrinkStorage = () => {
- try {
- storage.trim();
- } catch {}
+let dbPromise: Promise<SQLite.SQLiteDatabase> | null = null;
+
+const getDb = () => {
+ if (!dbPromise) {
+ dbPromise = SQLite.openDatabaseAsync(DATABASE_NAME)
+ .then(async (db) => {
+ await db.execAsync(
+ `PRAGMA journal_mode = WAL;
+ CREATE TABLE IF NOT EXISTS cache (
+ key TEXT PRIMARY KEY NOT NULL,
+ value TEXT NOT NULL
+ );`
+ );
+ deleteLegacyStorage();
+ return db;
+ })
+ .catch((e) => {
+ dbPromise = null;
+ throw e;
+ });
+ }
+
+ return dbPromise;
};
const infiniteQueryKeys = new Set(["links", "publicLinks", "tags"]);
@@ -29,18 +53,36 @@ const hasInfiniteDataShape = (data: unknown) =>
Array.isArray((data as any).pages) &&
Array.isArray((data as any).pageParams));
+const withoutStaleError = (query: any) => {
+ if (query?.state?.data === undefined) return query;
+ if (query.state.status === "success" && query.state.error == null) {
+ return query;
+ }
+
+ return {
+ ...query,
+ state: {
+ ...query.state,
+ status: "success",
+ error: null,
+ fetchFailureCount: 0,
+ fetchFailureReason: null,
+ },
+ };
+};
+
export const sanitizePersistedClient = (
client: PersistedClient
): PersistedClient => {
const queries = client.clientState?.queries;
if (!Array.isArray(queries)) return client;
- const sanitizedQueries = queries.filter((query) => {
- if (!isInfiniteQueryKey(query.queryKey)) return true;
- return hasInfiniteDataShape(query.state?.data);
- });
-
- if (sanitizedQueries.length === queries.length) return client;
+ const sanitizedQueries = queries
+ .filter((query) => {
+ if (!isInfiniteQueryKey(query.queryKey)) return true;
+ return hasInfiniteDataShape(query.state?.data);
+ })
+ .map(withoutStaleError);
return {
...client,
@@ -51,49 +93,95 @@ export const sanitizePersistedClient = (
};
};
-export const mmkvPersister: Persister = {
- persistClient: async (client) => {
- if (persistenceBroken) return;
+const WRITE_DEBOUNCE_MS = 3000;
- let json: string;
- try {
- json = JSON.stringify(client);
- } catch (e) {
- console.error("Error persisting client:", e);
- return;
- }
+let pendingClient: PersistedClient | null = null;
+let flushTimeout: ReturnType<typeof setTimeout> | null = null;
+let writeQueue: Promise<void> = Promise.resolve();
- try {
- storage.set(CACHE_KEY, json);
- } catch {
- try {
- storage.clearAll();
- storage.trim();
- storage.set(CACHE_KEY, json);
- } catch (e) {
- persistenceBroken = true;
- console.error(
- `Error persisting client (payload ${json.length} chars):`,
- e
- );
- }
- }
+const writePendingClient = async () => {
+ const client = pendingClient;
+ pendingClient = null;
+ if (!client) return;
+
+ let json: string;
+ try {
+ json = JSON.stringify(client);
+ } catch (e) {
+ console.error("Error persisting client:", e);
+ return;
+ }
+
+ try {
+ const db = await getDb();
+ await db.runAsync(
+ "INSERT OR REPLACE INTO cache (key, value) VALUES (?, ?)",
+ CACHE_KEY,
+ json
+ );
+ } catch (e) {
+ console.error(`Error persisting client (payload ${json.length} chars):`, e);
+ }
+};
+
+const cancelScheduledFlush = () => {
+ if (!flushTimeout) return;
+ clearTimeout(flushTimeout);
+ flushTimeout = null;
+};
+
+const enqueueWrite = () => {
+ writeQueue = writeQueue.then(writePendingClient).catch(() => {});
+ return writeQueue;
+};
+
+const scheduleFlush = () => {
+ if (flushTimeout) return;
+
+ flushTimeout = setTimeout(() => {
+ flushTimeout = null;
+ enqueueWrite();
+ }, WRITE_DEBOUNCE_MS);
+};
+
+const flushNow = () => {
+ cancelScheduledFlush();
+ return enqueueWrite();
+};
+
+AppState.addEventListener("change", (state) => {
+ if (state !== "active") flushNow();
+});
+
+export const queryPersister: Persister = {
+ persistClient: async (client) => {
+ pendingClient = client;
+ scheduleFlush();
},
restoreClient: async () => {
try {
- const json = storage.getString(CACHE_KEY);
- return json ? sanitizePersistedClient(JSON.parse(json)) : undefined;
+ const db = await getDb();
+ const row = await db.getFirstAsync<{ value: string }>(
+ "SELECT value FROM cache WHERE key = ?",
+ CACHE_KEY
+ );
+
+ return row?.value
+ ? sanitizePersistedClient(JSON.parse(row.value) as PersistedClient)
+ : undefined;
} catch (e) {
console.error("Error restoring client:", e);
return undefined;
- } finally {
- shrinkStorage();
}
},
removeClient: async () => {
+ pendingClient = null;
+ cancelScheduledFlush();
+ await writeQueue.catch(() => {});
+
try {
- storage.remove(CACHE_KEY);
- shrinkStorage();
+ const db = await getDb();
+ await db.runAsync("DELETE FROM cache WHERE key = ?", CACHE_KEY);
} catch (e) {
console.error("Error removing client:", e);
}
diff --git a/apps/mobile/package.json b/apps/mobile/package.json
index bef606e3..050eb062 100644
--- a/apps/mobile/package.json
+++ b/apps/mobile/package.json
@@ -52,6 +52,7 @@
"expo-secure-store": "~15.0.8",
"expo-share-intent": "^5.1.1",
"expo-splash-screen": "~31.0.13",
+ "expo-sqlite": "~16.0.10",
"expo-status-bar": "~3.0.9",
"expo-symbols": "~1.0.8",
"expo-system-ui": "~6.0.9",
@@ -70,8 +71,6 @@
"react-native-ios-context-menu": "^3.2.0",
"react-native-ios-utilities": "5.2.0",
"react-native-keyboard-controller": "1.20.7",
- "react-native-mmkv": "4.1.2",
- "react-native-nitro-modules": "0.34.1",
"react-native-pdf": "^7.0.3",
"react-native-reanimated": "4.1.1",
"react-native-safe-area-context": "^5.7.0",
diff --git a/apps/mobile/store/auth.ts b/apps/mobile/store/auth.ts
index 153831ec..44da59c3 100644
--- a/apps/mobile/store/auth.ts
+++ b/apps/mobile/store/auth.ts
@@ -12,7 +12,7 @@ import type { Config } from "@linkwarden/router/config";
import { Alert } from "react-native";
import { SheetManager } from "react-native-actions-sheet";
import { queryClient } from "@/lib/queryClient";
-import { mmkvPersister } from "@/lib/queryPersister";
+import { queryPersister } from "@/lib/queryPersister";
import { clearCache } from "@/lib/cache";
import useDataStore from "@/store/data";
import { markWhatsNewSeen } from "@/lib/whatsNew";
@@ -592,7 +592,7 @@ const useAuthStore = create<AuthStore>((set, get) => ({
queryClient.cancelQueries();
queryClient.clear();
- mmkvPersister.removeClient?.();
+ await queryPersister.removeClient?.();
await clearCache();
diff --git a/apps/mobile/store/data.ts b/apps/mobile/store/data.ts
index ef25c0ec..a9b17af8 100644
--- a/apps/mobile/store/data.ts
+++ b/apps/mobile/store/data.ts
@@ -29,9 +29,9 @@ const useDataStore = create<DataStore>((set, get) => ({
},
updateData: async (patch) => {
const merged = { ...get().data, ...patch };
+ set({ data: merged });
const { shareIntent, ...persistable } = merged;
await AsyncStorage.setItem("data", JSON.stringify(persistable));
- set({ data: merged });
},
}));
diff --git a/apps/web/pages/api/v1/archives/[linkId].ts b/apps/web/pages/api/v1/archives/[linkId].ts
index 6df3637b..b6d63840 100644
--- a/apps/web/pages/api/v1/archives/[linkId].ts
+++ b/apps/web/pages/api/v1/archives/[linkId].ts
@@ -17,7 +17,7 @@ import { ArchivedFormat } from "@linkwarden/types/global";
export const config = {
api: {
bodyParser: false,
- responseLimit: "50mb",
+ responseLimit: "100mb",
},
};
diff --git a/apps/web/pages/api/v1/preserved/view.ts b/apps/web/pages/api/v1/preserved/view.ts
index b31afabb..62f1b537 100644
--- a/apps/web/pages/api/v1/preserved/view.ts
+++ b/apps/web/pages/api/v1/preserved/view.ts
@@ -88,6 +88,12 @@ const getDownloadFilename = (format: ArchivedFormat, filePath: string) => {
}
};
+export const config = {
+ api: {
+ responseLimit: "100mb",
+ },
+};
+
export default async function handler(
req: NextApiRequest,
res: NextApiResponse
diff --git a/package.json b/package.json
index 4200f9d6..53c99144 100644
--- a/package.json
+++ b/package.json
@@ -35,8 +35,6 @@
},
"resolutions": {
"react": "19.1.0",
- "react-dom": "19.1.0",
- "react-native-mmkv": "4.1.2",
- "react-native-nitro-modules": "0.34.1"
+ "react-dom": "19.1.0"
}
}
diff --git a/packages/router/user.tsx b/packages/router/user.tsx
index b761c17e..e3925be9 100644
--- a/packages/router/user.tsx
+++ b/packages/router/user.tsx
@@ -35,7 +35,10 @@ const useUser = (auth?: MobileAuth) => {
: undefined
);
- if (!response.ok) throw new Error("Failed to fetch user data.");
+ if (!response.ok)
+ throw Object.assign(new Error("Failed to fetch user data."), {
+ status: response.status,
+ });
const data = (await response.json()).response as GetUserByIdResponse;
diff --git a/yarn.lock b/yarn.lock
index e578816d..0c6b6bcf 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5249,6 +5249,7 @@ __metadata:
expo-secure-store: "npm:~15.0.8"
expo-share-intent: "npm:^5.1.1"
expo-splash-screen: "npm:~31.0.13"
+ expo-sqlite: "npm:~16.0.10"
expo-status-bar: "npm:~3.0.9"
expo-symbols: "npm:~1.0.8"
expo-system-ui: "npm:~6.0.9"
@@ -5269,8 +5270,6 @@ __metadata:
react-native-ios-context-menu: "npm:^3.2.0"
react-native-ios-utilities: "npm:5.2.0"
react-native-keyboard-controller: "npm:1.20.7"
- react-native-mmkv: "npm:4.1.2"
- react-native-nitro-modules: "npm:0.34.1"
react-native-pdf: "npm:^7.0.3"
react-native-reanimated: "npm:4.1.1"
react-native-safe-area-context: "npm:^5.7.0"
@@ -10016,6 +10015,13 @@ __metadata:
languageName: node
linkType: hard
+"await-lock@npm:^2.2.2":
+ version: 2.2.2
+ resolution: "await-lock@npm:2.2.2"
+ checksum: 10c0/bedf00dad44c6325a655bf3bd523ab9e1ce41023da6a8c379990c76ac1d942ac7e5301627ab84ba37917ab5247506ba429b7f6e4bf77074093f255571b9ad5ee
+ languageName: node
+ linkType: hard
+
"axe-core@npm:^4.6.2":
version: 4.6.3
resolution: "axe-core@npm:4.6.3"
@@ -13053,6 +13059,19 @@ __metadata:
languageName: node
linkType: hard
+"expo-sqlite@npm:~16.0.10":
+ version: 16.0.10
+ resolution: "expo-sqlite@npm:16.0.10"
+ dependencies:
+ await-lock: "npm:^2.2.2"
+ peerDependencies:
+ expo: "*"
+ react: "*"
+ react-native: "*"
+ checksum: 10c0/7d686aa7305071d02a914d11581c763f83cd32c0c04b6a46852c458b81f96e2cbef4178e4526450b14a8bd684be7889ed8445e06cca038005082425b2900267f
+ languageName: node
+ linkType: hard
+
"expo-status-bar@npm:~3.0.9":
version: 3.0.9
resolution: "expo-status-bar@npm:3.0.9"
@@ -19215,27 +19234,6 @@ __metadata:
languageName: node
linkType: hard
-"react-native-mmkv@npm:4.1.2":
- version: 4.1.2
- resolution: "react-native-mmkv@npm:4.1.2"
- peerDependencies:
- react: "*"
- react-native: "*"
- react-native-nitro-modules: "*"
- checksum: 10c0/713f9c76e732fc010e53203f7dcb9fd571b1d89db13b8ea600ebc3b9505a3c8320e24acabdeb285e7bb0cc0c321d0374cf1a0cca9b211e0852555471aa90e396
- languageName: node
- linkType: hard
-
-"react-native-nitro-modules@npm:0.34.1":
- version: 0.34.1
- resolution: "react-native-nitro-modules@npm:0.34.1"
- peerDependencies:
- react: "*"
- react-native: "*"
- checksum: 10c0/1fbbf49cfd884fa9034f368ea3c989e16a3a6d56fd8001c8afe9839f83b772bc392a477cae962880b9728054a10247a93b1658dc3a8028bcf18032dfaac95a29
- languageName: node
- linkType: hard
-
"react-native-pdf@npm:^7.0.3":
version: 7.0.3
resolution: "react-native-pdf@npm:7.0.3"