AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.
HTML ai-agentcloudflarecloudflare-d1cloudflare-r2cloudflare-workersdockerelectronevernoteevernote-alternativeknowledge-basemcpnote-takingobsidian-alternativeoffline-firstpersonal-knowledge-basepwaself-hostedsqlitetiptapweb-clipper
fix: keep share and form links out of search indexes and caches (#608)
Dariusz Koryto82dac9b5cb14
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex a6ad9da2..7dcd0cc2 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -45,6 +45,7 @@ import {restoreJsonNotebooks as restoreJsonNotebooksService,} from "./backup-service";import { sha256, sha256Bytes } from "./hash-utils";+import { PUBLIC_PAGE_HEADERS } from "@edgeever/shared/public-page-headers";import { INSTANCE_BUILD_ID } from "./instance-build";import { resolveDeploymentVersionCreatedAt, resolveInstanceDeploymentMetadata } from "./instance-deployment";import type {@@ -175,6 +176,16 @@ app.use(}));+// Every anonymous share/form response, including errors, stays out of search+// indexes and shared caches. Routes may still set a stricter Cache-Control.+app.use("/api/public/*", async (c, next) => {+ await next();+ for (const [name, value] of Object.entries(PUBLIC_PAGE_HEADERS)) {+ if (name === "Cache-Control" && c.res.headers.has(name)) continue;+ c.res.headers.set(name, value);+ }+});+app.get("/api/release", (c) => c.json(releaseSummary));const getAppliedMigration = async (env: Bindings) => {diff --git a/apps/web/public/_headers b/apps/web/public/_headersindex d916b819..ea4d5d53 100644--- a/apps/web/public/_headers+++ b/apps/web/public/_headers@@ -23,3 +23,14 @@/assets/*Cache-Control: public, max-age=31536000, immutable++# Anonymous share and form links: the token in the URL is the only secret.+/share/*+ X-Robots-Tag: noindex, nofollow, noarchive, nosnippet, noimageindex+ Cache-Control: private, no-store+ Referrer-Policy: no-referrer++/form/*+ X-Robots-Tag: noindex, nofollow, noarchive, nosnippet, noimageindex+ Cache-Control: private, no-store+ Referrer-Policy: no-referrerdiff --git a/packages/shared/package.json b/packages/shared/package.jsonindex 7d3f014c..87046383 100644--- a/packages/shared/package.json+++ b/packages/shared/package.json@@ -24,7 +24,8 @@"./font-catalog": "./src/font-catalog.ts","./font-loader": "./src/font-loader.ts","./note-image-fonts": "./src/note-image-fonts.ts",- "./note-image-preferences": "./src/note-image-preferences.ts"+ "./note-image-preferences": "./src/note-image-preferences.ts",+ "./public-page-headers": "./src/public-page-headers.ts"},"types": "./src/index.ts","dependencies": {diff --git a/packages/shared/src/public-page-headers.test.mjs b/packages/shared/src/public-page-headers.test.mjsnew file mode 100644index 00000000..6f91422b--- /dev/null+++ b/packages/shared/src/public-page-headers.test.mjs@@ -0,0 +1,42 @@+import { describe, expect, test } from "bun:test";+import { readFileSync } from "node:fs";+import { isPublicPagePath, PUBLIC_PAGE_HEADERS } from "./public-page-headers.ts";++const readHeaderRules = (path) => {+ const rules = new Map();+ let current = null;+ for (const line of readFileSync(path, "utf8").split("\n")) {+ if (!line.trim() || line.trim().startsWith("#")) continue;+ if (!line.startsWith(" ")) {+ current = new Map();+ rules.set(line.trim(), current);+ continue;+ }+ const separator = line.indexOf(":");+ current.set(line.slice(0, separator).trim(), line.slice(separator + 1).trim());+ }+ return rules;+};++describe("public page headers", () => {+ test("matches share and form links only", () => {+ expect(isPublicPagePath("/share/abc")).toBe(true);+ expect(isPublicPagePath("/share/notebook/abc/memo")).toBe(true);+ expect(isPublicPagePath("/form/abc")).toBe(true);+ expect(isPublicPagePath("/")).toBe(false);+ expect(isPublicPagePath("/settings")).toBe(false);+ expect(isPublicPagePath("/shared")).toBe(false);+ });++ test("Cloudflare static asset rules send the same headers as the shared policy", () => {+ const rules = readHeaderRules(new URL("../../../apps/web/public/_headers", import.meta.url));+ for (const route of ["/share/*", "/form/*"]) {+ expect(Object.fromEntries(rules.get(route) ?? [])).toEqual(PUBLIC_PAGE_HEADERS);+ }+ });++ test("the Docker static server applies the policy to public app routes", () => {+ const server = readFileSync(new URL("../../../scripts/self-hosted-server.mjs", import.meta.url), "utf8");+ expect(server).toContain("...(isPublicPagePath(url.pathname) ? PUBLIC_PAGE_HEADERS : {})");+ });+});diff --git a/packages/shared/src/public-page-headers.ts b/packages/shared/src/public-page-headers.tsnew file mode 100644index 00000000..8b23f1ff--- /dev/null+++ b/packages/shared/src/public-page-headers.ts@@ -0,0 +1,16 @@+/**+ * Response headers for anonymous share and form links. The token in the URL is+ * the only secret, so these pages must stay out of search indexes, caches and+ * Referer headers sent to other sites.+ */+export const PUBLIC_PAGE_ROBOTS = "noindex, nofollow, noarchive, nosnippet, noimageindex";++export const PUBLIC_PAGE_HEADERS: Readonly<Record<string, string>> = {+ "X-Robots-Tag": PUBLIC_PAGE_ROBOTS,+ "Cache-Control": "private, no-store",+ "Referrer-Policy": "no-referrer",+};++/** Web routes rendered for anonymous visitors of a share or form link. */+export const isPublicPagePath = (pathname: string) =>+ pathname.startsWith("/share/") || pathname.startsWith("/form/");diff --git a/scripts/self-hosted-server.mjs b/scripts/self-hosted-server.mjsindex 76d3a71d..8ecbfbc5 100644--- a/scripts/self-hosted-server.mjs+++ b/scripts/self-hosted-server.mjs@@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url";import { hasBootstrapCredential } from "../apps/api/src/auth-bootstrap.ts";import { isUnauthenticatedAccessEnabled } from "../apps/api/src/auth-state.ts";import { fetchEdgeEverApp } from "../apps/api/src/index.ts";+import { isPublicPagePath, PUBLIC_PAGE_HEADERS } from "../packages/shared/src/public-page-headers.ts";import { nodePublicFetch } from "../apps/api/src/node-public-network.ts";import { createSelfHostedStorageAdapter } from "../apps/api/src/self-hosted-storage-adapter.ts";import { resolveSelfHostedConfig } from "./self-hosted-config.mjs";@@ -140,7 +141,13 @@ const serveStatic = async (request) => {const index = Bun.file(join(webDirectory, "index.html"));return (await index.exists())- ? new Response(index, { headers: { "Content-Type": "text/html; charset=utf-8" } })+ ? new Response(index, {+ headers: {+ "Content-Type": "text/html; charset=utf-8",+ // Share and form links render the app shell; keep them out of search indexes and caches.+ ...(isPublicPagePath(url.pathname) ? PUBLIC_PAGE_HEADERS : {}),+ },+ }): new Response("Web build not found. Run bun run build:web first.", { status: 503 });};diff --git a/tests/auth-fail-closed.test.ts b/tests/auth-fail-closed.test.tsindex aadd0095..a942a63f 100644--- a/tests/auth-fail-closed.test.ts+++ b/tests/auth-fail-closed.test.ts@@ -101,6 +101,21 @@ describe("production authentication guard", () => {});});+ test("keeps anonymous share responses out of search indexes, caches and Referer headers", async () => {+ for (const path of ["/api/public/shares/not-a-token", "/api/public/notebook-shares/not-a-token", "/api/public/forms/not-a-token"]) {+ const response = await fetchApi(path, {+ DB: createDatabase({ userId: "owner" }),+ RESOURCES: {},+ EDGE_EVER_AUTH_PASSWORD: "configured-secret",+ });++ expect(response.status).toBe(404);+ expect(response.headers.get("X-Robots-Tag")).toBe("noindex, nofollow, noarchive, nosnippet, noimageindex");+ expect(response.headers.get("Cache-Control")).toBe("private, no-store");+ expect(response.headers.get("Referrer-Policy")).toBe("no-referrer");+ }+ });+test("does not misreport transient D1 failures as unapplied migrations", async () => {const response = await fetchApi("/api/health", {DB: createDatabase({ error: new Error("D1_ERROR: Network connection lost.") }),