Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
fix(sync): stop clock-ahead edits winning every conflict (#12173)
Elian Doranf26b835c26ae
diff --git a/packages/trilium-core/src/services/entity_changes.ts b/packages/trilium-core/src/services/entity_changes.tsindex 5f4807ace7..5435cb3997 100644--- a/packages/trilium-core/src/services/entity_changes.ts+++ b/packages/trilium-core/src/services/entity_changes.ts@@ -97,6 +97,24 @@ function putBlobEntityChange(blobId: string) {} as EntityChange);}+/**+ * Moves every entity change dated later than now back to now. Such a change was written while the+ * clock ran ahead, and sync resolves conflicts by `utcDateChanged`, so it would otherwise win+ * against every real edit made before the clock catches up with it.+ */+function restampFutureEntityChanges() {+ const now = dateUtils.utcNowDateTime();+ const sql = getSql();+ const count = sql.getValue<number>("SELECT COUNT(*) FROM entity_changes WHERE utcDateChanged > ?", [now]) ?? 0;++ if (count === 0) {+ return;+ }++ sql.execute("UPDATE entity_changes SET utcDateChanged = ? WHERE utcDateChanged > ?", [now, now]);+ getLog().info(`Moved ${count} entity changes dated in the future back to ${now}`);+}+function putNoteReorderingEntityChange(parentNoteId: string, componentId?: string) {putEntityChange({entityName: "note_reordering",@@ -259,5 +277,6 @@ export default {fillAllEntityChanges,addEntityChangesForSector,getMaxEntityChangeId: () => maxEntityChangeId,- recalculateMaxEntityChangeId+ recalculateMaxEntityChangeId,+ restampFutureEntityChanges};diff --git a/packages/trilium-core/src/services/sync.spec.ts b/packages/trilium-core/src/services/sync.spec.tsindex 9ac7cefaf6..3aa5c6d5d8 100644--- a/packages/trilium-core/src/services/sync.spec.ts+++ b/packages/trilium-core/src/services/sync.spec.ts@@ -45,12 +45,14 @@ let changedIdx = 0;let checkIdx = 0;const requestLog: Array<{ method: string; url: string; body?: ExecOpts["body"] }> = [];-/** The entity IDs actually sent to the sync server, across every push of the run. */-const pushedEntityIds = () => requestLog+/** The records actually sent to the sync server, across every push of the run. */+const pushedRecords = () => requestLog.filter((r) => r.url.includes("/api/sync/update"))// syncRequest always serializes the body itself, so a push body is a JSON string.- .flatMap((r) => (typeof r.body === "string" ? JSON.parse(r.body || "{}").entities ?? [] : []) as EntityChangeRecord[])- .map((record) => record.entityChange.entityId);+ .flatMap((r) => (typeof r.body === "string" ? JSON.parse(r.body || "{}").entities ?? [] : []) as EntityChangeRecord[]);++/** The entity IDs actually sent to the sync server, across every push of the run. */+const pushedEntityIds = () => pushedRecords().map((record) => record.entityChange.entityId);const fakeRequest: RequestProvider = fakeRequestProvider({exec: (<T,>(opts: ExecOpts): Promise<T> => {@@ -353,20 +355,16 @@ describe("sync service", () => {expect(pushedEntityIds()).toContain(blobId);});- // The client stamped a note while its clock ran ahead, the server edited the note- // afterwards and dropped the client's blob in `deleteBlobIfNotUsed()`. The client's row wins- // the pull, so the push must carry the blob that row points at.- it("pushes the blob of a local note that wins against a newer server edit", async () => {- const clockAhead = dateUtils.utcDateTimeStr(new Date(Date.now() + 3600 * 1000));+ // The server edited the note and dropped the client's blob in `deleteBlobIfNotUsed()`. The+ // client's later row wins the pull, so the push must carry the blob that row points at.+ it("pushes the blob of a local note that wins against an older server edit", async () => {const { noteId, clientBlobId } = cls.init(() => {- const { note } = notes.createNewNote({ parentNoteId: "root", title: "12173", content: "<p>client text</p>", type: "text" });- const sql = getSql();- sql.execute("UPDATE entity_changes SET utcDateChanged = ? WHERE entityName = 'notes' AND entityId = ?", [clockAhead, note.noteId]);+ const { note } = notes.createNewNote({ parentNoteId: "root", title: "local-wins", content: "<p>client text</p>", type: "text" });options.setOption("lastSyncedPush", String(syncService.getMaxEntityChangeId()));return { noteId: note.noteId, clientBlobId: note.blobId };});- const serverBlobId = "su12173ServerEdit";- const now = dateUtils.utcNowDateTime();+ const serverBlobId = "suLocalWinsServerEdit";+ const now = dateUtils.utcDateTimeStr(new Date(Date.now() - 60 * 1000));const clientRow = getSql().getRow<Record<string, unknown>>("SELECT * FROM notes WHERE noteId = ?", [noteId]);config.changed = [@@ -387,6 +385,39 @@ describe("sync service", () => {expect(pushedEntityIds()).toContain(clientBlobId);});+ it("lets a server edit made after the sync beat a note stamped while the clock ran ahead", async () => {+ const clockAhead = dateUtils.utcDateTimeStr(new Date(Date.now() + 3600 * 1000));+ const noteId = cls.init(() => {+ const { note } = notes.createNewNote({ parentNoteId: "root", title: "clock-ahead", content: "<p>client text</p>", type: "text" });+ getSql().execute("UPDATE entity_changes SET utcDateChanged = ? WHERE entityName = 'notes' AND entityId = ?", [clockAhead, note.noteId]);+ return note.noteId;+ });++ await expect(runSync()).resolves.toEqual({ success: true });+ const pushedNote = pushedRecords().find((record) => record.entityChange.entityId === noteId);+ expect(pushedNote).toBeDefined();+ expect.soft((pushedNote?.entityChange.utcDateChanged ?? clockAhead) < clockAhead).toBe(true);++ const serverBlobId = "suClockAheadServerEdit";+ const later = dateUtils.utcNowDateTime();+ const clientRow = getSql().getRow<Record<string, unknown>>("SELECT * FROM notes WHERE noteId = ?", [noteId]);+ changedIdx = 0;+ config.changed = [+ {+ entityChanges: [+ { entityChange: ec({ entityName: "blobs", entityId: serverBlobId, utcDateChanged: later }), entity: { blobId: serverBlobId, content: encodeBase64("<p>server text</p>"), dateModified: later, utcDateModified: later } },+ { entityChange: ec({ entityId: noteId, hash: "serverEdit", utcDateChanged: later }), entity: { ...clientRow, blobId: serverBlobId, utcDateModified: later } }+ ],+ lastEntityChangeId: 2,+ outstandingPullCount: 0+ },+ { entityChanges: [], lastEntityChangeId: 2, outstandingPullCount: 0 }+ ];++ await expect(runSync()).resolves.toEqual({ success: true });+ expect(getSql().getValue("SELECT blobId FROM notes WHERE noteId = ?", [noteId])).toBe(serverBlobId);+ });+describe("unresolvable content hash divergence", () => {// A sector that keeps diverging after being re-queued used to spin the sync loop forever,// re-pushing the same sector every round without ever converging (#11073).diff --git a/packages/trilium-core/src/services/sync.ts b/packages/trilium-core/src/services/sync.tsindex e50bc7a018..e049845855 100644--- a/packages/trilium-core/src/services/sync.ts+++ b/packages/trilium-core/src/services/sync.ts@@ -111,6 +111,9 @@ async function sync() {const syncContext = await login();+ // The login succeeded, so the clock agrees with the sync server's.+ entityChangesService.restampFutureEntityChanges();+await pushChanges(syncContext);await pullChanges(syncContext);