Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

server(themes): poll for theme updates on every replica instead of using /sync

Abdullah Atta4e01f4cd650a
diff --git a/servers/themes/src/api.ts b/servers/themes/src/api.ts
index fd2ea2e1b..9a9a9d175 100644
--- a/servers/themes/src/api.ts
+++ b/servers/themes/src/api.ts
@@ -20,12 +20,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { z } from "zod";
import { InstallsCounter } from "./constants";
import { findTheme, getThemes, updateTotalInstalls } from "./orama";
-import { syncThemes } from "./sync";
import { publicProcedure, router } from "./trpc";
import { THEME_COMPATIBILITY_VERSION } from "@notesnook/theme";
import { ThemeQuerySchema } from "./schemas";
-import { TRPCError } from "@trpc/server";
-import { SYNC_TOKEN } from "./constants";
export const ThemesAPI = router({
themes: publicProcedure.input(ThemeQuerySchema).query(async ({ input }) => {
@@ -63,12 +60,6 @@ export const ThemesAPI = router({
const theme = await findTheme(id, compatibilityVersion);
if (theme && theme.version !== version) return theme;
}),
- sync: publicProcedure.query(({ ctx }) => {
- if (SYNC_TOKEN && ctx.authorization !== `Bearer ${SYNC_TOKEN}`)
- throw new TRPCError({ code: "UNAUTHORIZED" });
- syncThemes().catch((e) => console.error("Failed to sync themes:", e));
- return true;
- }),
health: publicProcedure.query(() => {
return "Healthy";
})
diff --git a/servers/themes/src/constants.ts b/servers/themes/src/constants.ts
index 67d20530d..2d49ac323 100644
--- a/servers/themes/src/constants.ts
+++ b/servers/themes/src/constants.ts
@@ -26,12 +26,9 @@ import { KVCounter } from "./counter/kv";
const env = readSecrets([
"CLOUDFLARE_ACCOUNT_ID",
"CLOUDFLARE_AUTH_TOKEN",
- "CLOUDFLARE_INSTALLS_KV_NAMESPACE_ID",
- "THEMES_SYNC_TOKEN"
+ "CLOUDFLARE_INSTALLS_KV_NAMESPACE_ID"
]);
-export const SYNC_TOKEN = env.THEMES_SYNC_TOKEN;
-
const __dirname = path.dirname(fileURLToPath(import.meta.url));
export const THEMES_REPO_URL =
diff --git a/servers/themes/src/server.ts b/servers/themes/src/server.ts
index 4a5f37e29..ca5040083 100644
--- a/servers/themes/src/server.ts
+++ b/servers/themes/src/server.ts
@@ -21,22 +21,25 @@ import { createHTTPServer } from "@trpc/server/adapters/standalone";
import { ThemesAPI } from "./api";
import { syncThemes } from "./sync";
import cors from "cors";
-import { createContext } from "./trpc";
const server = createHTTPServer({
middleware: cors(),
- router: ThemesAPI,
- createContext
+ router: ThemesAPI
});
const PORT = parseInt(process.env.PORT || "9000");
const HOST = process.env.HOST || "localhost";
server.listen(PORT, HOST);
console.log(`Server started successfully on: http://${HOST}:${PORT}/`);
-function sync() {
- syncThemes().catch((e) => console.error("Failed to sync themes:", e));
+function sync(onlyIfChanged?: boolean) {
+ syncThemes(onlyIfChanged).catch((e) =>
+ console.error("Failed to sync themes:", e)
+ );
}
sync();
+// each replica polls on its own since a webhook would only reach one of them
+setInterval(() => sync(true), 1000 * 60); // every minute
+// full rebuild also refreshes install counts written by other replicas
setInterval(sync, 1000 * 60 * 60); // every hour
if (import.meta.hot) {
diff --git a/servers/themes/src/sync.ts b/servers/themes/src/sync.ts
index 24f4f4d64..d3592c756 100644
--- a/servers/themes/src/sync.ts
+++ b/servers/themes/src/sync.ts
@@ -52,28 +52,36 @@ const execFileAsync = promisify(execFile);
const GIT_TIMEOUT = 60 * 1000;
let currentSync: Promise<void> | undefined;
-export function syncThemes() {
+export function syncThemes(onlyIfChanged = false) {
// concurrent git pulls fail on index.lock, so piggyback on the running sync
if (!currentSync)
- currentSync = sync().finally(() => (currentSync = undefined));
+ currentSync = sync(onlyIfChanged).finally(() => (currentSync = undefined));
return currentSync;
}
async function git(args: string[], cwd: string) {
- const { stdout, stderr } = await execFileAsync("git", args, {
+ const { stdout } = await execFileAsync("git", args, {
cwd,
timeout: GIT_TIMEOUT
});
- if (stderr.trim()) console.log(stderr.trim());
- if (stdout.trim()) console.log(stdout.trim());
+ return stdout.trim();
}
-async function sync() {
+async function hasUpstreamChanges() {
+ const [remote, local] = await Promise.all([
+ git(["ls-remote", "origin", "HEAD"], THEME_REPO_DIR_PATH),
+ git(["rev-parse", "HEAD"], THEME_REPO_DIR_PATH)
+ ]);
+ return remote.split(/\s/)[0] !== local;
+}
+
+async function sync(onlyIfChanged: boolean) {
if (!fs.existsSync(THEME_REPO_DIR_PATH)) {
await git(["clone", THEMES_REPO_URL], path.dirname(THEME_REPO_DIR_PATH));
console.log(`Cloned github repo to path ${THEME_REPO_DIR_PATH}`);
} else {
- await git(["pull"], THEME_REPO_DIR_PATH);
+ if (onlyIfChanged && !(await hasUpstreamChanges())) return;
+ console.log(await git(["pull"], THEME_REPO_DIR_PATH));
console.log(`Synced github repo ${THEMES_REPO_URL}`);
}
await generateThemesMetadata();
diff --git a/servers/themes/src/trpc.ts b/servers/themes/src/trpc.ts
index 6e387a974..c76d638f5 100644
--- a/servers/themes/src/trpc.ts
+++ b/servers/themes/src/trpc.ts
@@ -18,17 +18,12 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { initTRPC } from "@trpc/server";
-import type { CreateHTTPContextOptions } from "@trpc/server/adapters/standalone";
/**
* Initialization of tRPC backend
* Should be done only once per backend!
*/
-export function createContext({ req }: CreateHTTPContextOptions) {
- return { authorization: req.headers.authorization };
-}
-
-const t = initTRPC.context<typeof createContext>().create();
+const t = initTRPC.create();
/**
* Export reusable router and procedure helpers