A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
server(themes): poll for theme updates on every replica instead of using /sync
Abdullah Atta4e01f4cd650a
diff --git a/servers/themes/src/api.ts b/servers/themes/src/api.tsindex fd2ea2e1b..9a9a9d175 100644--- a/servers/themes/src/api.ts+++ b/servers/themes/src/api.ts@@ -20,12 +20,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { z } from "zod";import { InstallsCounter } from "./constants";import { findTheme, getThemes, updateTotalInstalls } from "./orama";-import { syncThemes } from "./sync";import { publicProcedure, router } from "./trpc";import { THEME_COMPATIBILITY_VERSION } from "@notesnook/theme";import { ThemeQuerySchema } from "./schemas";-import { TRPCError } from "@trpc/server";-import { SYNC_TOKEN } from "./constants";export const ThemesAPI = router({themes: publicProcedure.input(ThemeQuerySchema).query(async ({ input }) => {@@ -63,12 +60,6 @@ export const ThemesAPI = router({const theme = await findTheme(id, compatibilityVersion);if (theme && theme.version !== version) return theme;}),- sync: publicProcedure.query(({ ctx }) => {- if (SYNC_TOKEN && ctx.authorization !== `Bearer ${SYNC_TOKEN}`)- throw new TRPCError({ code: "UNAUTHORIZED" });- syncThemes().catch((e) => console.error("Failed to sync themes:", e));- return true;- }),health: publicProcedure.query(() => {return "Healthy";})diff --git a/servers/themes/src/constants.ts b/servers/themes/src/constants.tsindex 67d20530d..2d49ac323 100644--- a/servers/themes/src/constants.ts+++ b/servers/themes/src/constants.ts@@ -26,12 +26,9 @@ import { KVCounter } from "./counter/kv";const env = readSecrets(["CLOUDFLARE_ACCOUNT_ID","CLOUDFLARE_AUTH_TOKEN",- "CLOUDFLARE_INSTALLS_KV_NAMESPACE_ID",- "THEMES_SYNC_TOKEN"+ "CLOUDFLARE_INSTALLS_KV_NAMESPACE_ID"]);-export const SYNC_TOKEN = env.THEMES_SYNC_TOKEN;-const __dirname = path.dirname(fileURLToPath(import.meta.url));export const THEMES_REPO_URL =diff --git a/servers/themes/src/server.ts b/servers/themes/src/server.tsindex 4a5f37e29..ca5040083 100644--- a/servers/themes/src/server.ts+++ b/servers/themes/src/server.ts@@ -21,22 +21,25 @@ import { createHTTPServer } from "@trpc/server/adapters/standalone";import { ThemesAPI } from "./api";import { syncThemes } from "./sync";import cors from "cors";-import { createContext } from "./trpc";const server = createHTTPServer({middleware: cors(),- router: ThemesAPI,- createContext+ router: ThemesAPI});const PORT = parseInt(process.env.PORT || "9000");const HOST = process.env.HOST || "localhost";server.listen(PORT, HOST);console.log(`Server started successfully on: http://${HOST}:${PORT}/`);-function sync() {- syncThemes().catch((e) => console.error("Failed to sync themes:", e));+function sync(onlyIfChanged?: boolean) {+ syncThemes(onlyIfChanged).catch((e) =>+ console.error("Failed to sync themes:", e)+ );}sync();+// each replica polls on its own since a webhook would only reach one of them+setInterval(() => sync(true), 1000 * 60); // every minute+// full rebuild also refreshes install counts written by other replicassetInterval(sync, 1000 * 60 * 60); // every hourif (import.meta.hot) {diff --git a/servers/themes/src/sync.ts b/servers/themes/src/sync.tsindex 24f4f4d64..d3592c756 100644--- a/servers/themes/src/sync.ts+++ b/servers/themes/src/sync.ts@@ -52,28 +52,36 @@ const execFileAsync = promisify(execFile);const GIT_TIMEOUT = 60 * 1000;let currentSync: Promise<void> | undefined;-export function syncThemes() {+export function syncThemes(onlyIfChanged = false) {// concurrent git pulls fail on index.lock, so piggyback on the running syncif (!currentSync)- currentSync = sync().finally(() => (currentSync = undefined));+ currentSync = sync(onlyIfChanged).finally(() => (currentSync = undefined));return currentSync;}async function git(args: string[], cwd: string) {- const { stdout, stderr } = await execFileAsync("git", args, {+ const { stdout } = await execFileAsync("git", args, {cwd,timeout: GIT_TIMEOUT});- if (stderr.trim()) console.log(stderr.trim());- if (stdout.trim()) console.log(stdout.trim());+ return stdout.trim();}-async function sync() {+async function hasUpstreamChanges() {+ const [remote, local] = await Promise.all([+ git(["ls-remote", "origin", "HEAD"], THEME_REPO_DIR_PATH),+ git(["rev-parse", "HEAD"], THEME_REPO_DIR_PATH)+ ]);+ return remote.split(/\s/)[0] !== local;+}++async function sync(onlyIfChanged: boolean) {if (!fs.existsSync(THEME_REPO_DIR_PATH)) {await git(["clone", THEMES_REPO_URL], path.dirname(THEME_REPO_DIR_PATH));console.log(`Cloned github repo to path ${THEME_REPO_DIR_PATH}`);} else {- await git(["pull"], THEME_REPO_DIR_PATH);+ if (onlyIfChanged && !(await hasUpstreamChanges())) return;+ console.log(await git(["pull"], THEME_REPO_DIR_PATH));console.log(`Synced github repo ${THEMES_REPO_URL}`);}await generateThemesMetadata();diff --git a/servers/themes/src/trpc.ts b/servers/themes/src/trpc.tsindex 6e387a974..c76d638f5 100644--- a/servers/themes/src/trpc.ts+++ b/servers/themes/src/trpc.ts@@ -18,17 +18,12 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import { initTRPC } from "@trpc/server";-import type { CreateHTTPContextOptions } from "@trpc/server/adapters/standalone";/*** Initialization of tRPC backend* Should be done only once per backend!*/-export function createContext({ req }: CreateHTTPContextOptions) {- return { authorization: req.headers.authorization };-}--const t = initTRPC.context<typeof createContext>().create();+const t = initTRPC.create();/*** Export reusable router and procedure helpers