Skip to content

armgit / kiln

Public Imported from GitHub
0

Build and revise procedural 3D assets with your coding agent. Local MCP server, CLI and TypeScript engine with rendered review, editable source and GLB export.

feat(site): date sitemap URLs and keep pages.dev hosts out of search (#278)

Matthew Kissinger6ff8f8ab89cd
diff --git a/docs/site-indexing.md b/docs/site-indexing.md
index b7df527d..1660a944 100644
--- a/docs/site-indexing.md
+++ b/docs/site-indexing.md
@@ -60,3 +60,23 @@ returned 200. The existing sitemap index was resubmitted in Search Console, whic
confirmed **Sitemap submitted successfully** and then **Success**, with its last
read on September 6. Discovered pages initially remained zero; processing and
indexing are separate from successful retrieval. No account permissions changed.
+
+## Sitemap dates and pages.dev hosts — October 26, 2026
+
+Live checks against build `238d1aa8` (the `main` head): all 329 sitemap URLs returned
+200 with a self-referencing canonical on `https://kilnstudio.tools` and no `noindex`;
+every share image returned 200 at 1200×630. The sitemap carried no `<lastmod>`, and
+`kilnstudio.pages.dev` served the full site with no `X-Robots-Tag`. Preview aliases
+already answered with `X-Robots-Tag: noindex`.
+
+- `finalize-site` now writes a `<lastmod>` for every sitemap URL
+ (`site/scripts/sitemap-lastmod.mjs`): the newest commit date among the files the
+ page is built from (its Astro page, the local modules and data records it imports,
+ and for a guide its Markdown). The shared base layout and site constants are not
+ followed, so a navigation or version-label change does not redate every page.
+ `validate-static` fails a sitemap URL without exactly one W3C Datetime `<lastmod>`.
+- `_headers` sends `X-Robots-Tag: noindex` on `kilnstudio.pages.dev` and every
+ `*.kilnstudio.pages.dev` alias, following Cloudflare's documented absolute-URL rule.
+ The production host is unaffected; `validate-static` checks both.
+
+Robots rules, canonicals and the noindex pages are unchanged.
diff --git a/site/public/_headers b/site/public/_headers
index 5282cec9..425ea5db 100644
--- a/site/public/_headers
+++ b/site/public/_headers
@@ -17,6 +17,15 @@
Cache-Control: public, max-age=0, must-revalidate
Content-Security-Policy-Report-Only: default-src 'self'; script-src 'self' 'wasm-unsafe-eval' 'sha256-wAGukfde07th8FeXAlx31NqovrboToJ/niuiFbyoI70=' 'sha256-nQyK+pVXDH5RpnnZ8bk4KjsgiC91NvDr20Qc7mo8J4w=' 'sha256-qsHLNpEk33lPgo1MnVDl/NNWHbeSVPK4+ksw8A84/ls=' 'sha256-pi2avave5D7tqBWNSP54Cy8gsoNF+VSK2JuekU4lFcg=' 'sha256-ynl5VnLFfq74TN1iDcyssM5zkT3yce8NSUQIazX5/GY='; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; connect-src 'self' blob: https://assets.kilnstudio.tools; worker-src blob:
+# The project's own pages.dev hosts serve these same bytes (production at kilnstudio.pages.dev, each deployment and
+# branch alias beneath it), so they are kept out of search indexes; only https://kilnstudio.tools is indexed. Pages
+# matches an absolute rule on its host, where ":name" is one label (developers.cloudflare.com/pages/configuration/headers/).
+https://kilnstudio.pages.dev/*
+ X-Robots-Tag: noindex
+
+https://:version.kilnstudio.pages.dev/*
+ X-Robots-Tag: noindex
+
# Build output named by content hash: a changed file gets a new name, so these never need revalidating.
/_astro/*
! Cache-Control
diff --git a/site/scripts/finalize-site.mjs b/site/scripts/finalize-site.mjs
index 744c3c57..ffd204d3 100644
--- a/site/scripts/finalize-site.mjs
+++ b/site/scripts/finalize-site.mjs
@@ -3,6 +3,7 @@ import { dirname, relative, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { ARCHIVE_INDEX_ROUTE, inspectHtml, isArchiveItemRoute, ORIGIN, routeForFile, scriptHashSource } from './static-validation-core.mjs';
import {sealedTroyCache} from './sealed-cache.mjs';
+import { sitemapLastmod } from './sitemap-lastmod.mjs';
const escapeXml = (text) => text.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;');
const link = (page) => `- [${page.title.replace(/ · Archive · Kiln$| · Kiln docs$| · Kiln$| — Kiln$| — source and 3D model \| Kiln$| — Kiln Commons$/g, '')}](${new URL(page.route, ORIGIN).href}): ${page.description ?? ''}`;
@@ -22,8 +23,8 @@ export function buildHeaders(template, pages) {
return text;
}
-/** Derive discovery files from the pages that were actually emitted by Astro. */
-export function buildIndexFiles(pages) {
+/** Derive discovery files from the pages that were actually emitted by Astro. `lastmod(route)` dates each sitemap URL. */
+export function buildIndexFiles(pages, { lastmod } = {}) {
// Archive item pages are excluded by route as well as by their noindex marker: the marker is the source of truth,
// and validate-static fails a build in which an item lost it, but the discovery files never depend on that alone.
const published = pages.filter((page) => !page.noindex && page.route !== '/404.html' && !isArchiveItemRoute(page.route)).sort((a, b) => a.route.localeCompare(b.route));
@@ -31,7 +32,10 @@ export function buildIndexFiles(pages) {
const docs = published.filter((page) => page.route.startsWith('/docs/'));
const catalog = published.filter((page) => /^\/(?:packs|gallery|scenes)\/(?:[^/]+\/)?$/.test(page.route) && page.route !== ARCHIVE_INDEX_ROUTE);
const archive = published.find((page) => page.route === ARCHIVE_INDEX_ROUTE);
- const sitemap = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${published.map((page) => ` <url><loc>${escapeXml(new URL(page.route, ORIGIN).href)}</loc></url>`).join('\n')}\n</urlset>\n`;
+ const sitemap = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${published.map((page) => {
+ const date = lastmod?.(page.route);
+ return ` <url><loc>${escapeXml(new URL(page.route, ORIGIN).href)}</loc>${date ? `<lastmod>${escapeXml(date)}</lastmod>` : ''}</url>`;
+ }).join('\n')}\n</urlset>\n`;
// The owner's Search Console holds a submission of /sitemap-index.xml from the previous site (docs/site-indexing.md), so the path
// keeps answering with an index of the one sitemap. It is generated with the sitemap, never a checked-in copy that could drift.
const sitemapIndex = `<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n <sitemap><loc>${ORIGIN}/sitemap.xml</loc></sitemap>\n</sitemapindex>\n`;
@@ -70,7 +74,7 @@ export async function finalizeSite({ dist = resolve(dirname(fileURLToPath(import
pages.push({ route: routeForFile(relative(dist, file)), ...inspectHtml(await readFile(file, 'utf8')) });
}
if (!pages.some((page) => page.route === '/')) throw new Error(`Astro home output missing in ${dist}; build the site before finalizing.`);
- const { sitemap, sitemapIndex, robots, llms } = buildIndexFiles(pages);
+ const { sitemap, sitemapIndex, robots, llms } = buildIndexFiles(pages, { lastmod: sitemapLastmod() });
const headerPath=resolve(dist,'_headers');
const sealedCache=await sealedTroyCache(dist);
const headers=buildHeaders(await readFile(headerPath,'utf8'),pages)+sealedCache.headers;
diff --git a/site/scripts/finalize-site.test.ts b/site/scripts/finalize-site.test.ts
index 9f6b0f85..69821a15 100644
--- a/site/scripts/finalize-site.test.ts
+++ b/site/scripts/finalize-site.test.ts
@@ -88,3 +88,11 @@ test('a sitemap index lists the sitemap and only the sitemap', () => {
expect(sitemapIndex).toBe('<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n <sitemap><loc>https://kilnstudio.tools/sitemap.xml</loc></sitemap>\n</sitemapindex>\n');
expect(locs(sitemapIndex)).toEqual(['https://kilnstudio.tools/sitemap.xml']);
});
+
+test('each sitemap URL carries the lastmod its route is dated with', () => {
+ const lastmod = (route: string) => (route === '/' ? '2026-10-25T22:07:37-04:00' : '2026-09-01T10:00:00Z');
+ const { sitemap } = buildIndexFiles(pages, { lastmod });
+ expect(sitemap).toContain('<url><loc>https://kilnstudio.tools/</loc><lastmod>2026-10-25T22:07:37-04:00</lastmod></url>');
+ expect(sitemap).toContain('<url><loc>https://kilnstudio.tools/docs/install/</loc><lastmod>2026-09-01T10:00:00Z</lastmod></url>');
+ expect(locs(sitemap).length).toBe((sitemap.match(/<lastmod>/g) ?? []).length);
+});
diff --git a/site/scripts/sitemap-lastmod.mjs b/site/scripts/sitemap-lastmod.mjs
new file mode 100644
index 00000000..7dc67873
--- /dev/null
+++ b/site/scripts/sitemap-lastmod.mjs
@@ -0,0 +1,117 @@
+/**
+ * `<lastmod>` for each sitemap URL: the committer date of the newest commit that touched a file the page is built from.
+ *
+ * Google uses lastmod only when it is "consistently and verifiably accurate"
+ * (https://developers.google.com/search/docs/crawling-indexing/sitemaps/build-sitemap), so the date is derived, never
+ * typed: a route is matched to its Astro page file, the page's local imports are followed (data records, components,
+ * layouts, libraries; stylesheets are skipped because a CSS change is not a content change), and a documentation route
+ * adds its repository Markdown (from `KILN_SITE_DOCS_DIR`, as the docs loader reads it). The newest Git date among
+ * those files is the page's lastmod. A file with no commit yet (an uncommitted addition) counts as changed at HEAD.
+ * The shared chrome every page carries (`SITE_CHROME`: the base layout's header, footer and share card, and the
+ * site constants with the release label) is not followed: Google asks for the last *significant* change to a page, and
+ * a navigation link or version label changing on all pages at once would otherwise redate the whole sitemap.
+ * Files read at build time from untracked inputs (the restored example archive, generated sources) are pinned by
+ * tracked records, which are the files that carry the date.
+ */
+import { execFileSync } from 'node:child_process';
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { dirname, join, relative, resolve, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { docsDirectory } from './source-dirs.mjs';
+
+const SITE = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const EXTENSIONS = ['', '.ts', '.tsx', '.mjs', '.js', '.astro', '.json'];
+const SOURCE = /\.(?:astro|[cm]?[jt]sx?|json)$/;
+/** Repository paths of the site-wide chrome that no page's lastmod follows (see above). */
+export const SITE_CHROME = ['site/src/layouts/BaseLayout.astro', 'site/src/lib/config.ts'];
+const IMPORT = /(?:^|[\s;])(?:import|export)\s+(?:[^'"`;]*?\sfrom\s*)?['"]([^'"]+)['"]|\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
+
+/** The Astro page pattern for a file under src/pages: static segments, `[name]` (one segment) or `[...name]` (any). */
+export function pagePattern(file) {
+ const path = file.replaceAll('\\', '/').replace(/\.astro$/, '').replace(/(?:^|\/)index$/, '');
+ return path ? path.split('/') : [];
+}
+
+/** The page file that emitted `route`, by Astro's priority: more static segments first, then single params over rest. */
+export function pageForRoute(route, files) {
+ const segments = route.replace(/\.html$/, '').split('/').filter(Boolean);
+ const matches = (pattern, at = 0, index = 0) => {
+ if (index === pattern.length) return at === segments.length;
+ const part = pattern[index];
+ if (/^\[\.\.\.[^\]]+\]$/.test(part)) {
+ for (let end = segments.length; end >= at; end--) if (matches(pattern, end, index + 1)) return true;
+ return false;
+ }
+ if (at >= segments.length) return false;
+ return (/^\[[^\]]+\]$/.test(part) || part === segments[at]) && matches(pattern, at + 1, index + 1);
+ };
+ const score = (pattern) => [pattern.filter((part) => !part.startsWith('[')).length, -pattern.filter((part) => part.startsWith('[...')).length];
+ let best;
+ for (const file of files) {
+ const pattern = pagePattern(file);
+ if (!matches(pattern)) continue;
+ const [fixed, rest] = score(pattern);
+ if (!best || fixed > best.fixed || (fixed === best.fixed && rest > best.rest)) best = { file, fixed, rest };
+ }
+ return best?.file;
+}
+
+/** Every local file reachable from `entry` through relative imports, `entry` included, except files `skip` refuses. */
+export function sourceClosure(entry, { read = (file) => readFileSync(file, 'utf8'), isFile = (file) => existsSync(file) && statSync(file).isFile(), skip = () => false } = {}) {
+ const seen = new Set();
+ const visit = (file) => {
+ if (seen.has(file) || (file !== entry && skip(file))) return;
+ seen.add(file);
+ if (!SOURCE.test(file) || file.endsWith('.json')) return;
+ for (const match of read(file).matchAll(IMPORT)) {
+ const specifier = (match[1] ?? match[2]).split('?')[0];
+ if (!specifier.startsWith('.') || specifier.endsWith('.css')) continue;
+ const base = resolve(dirname(file), specifier);
+ const target = [...EXTENSIONS.map((extension) => base + extension), ...EXTENSIONS.slice(1).map((extension) => join(base, `index${extension}`))]
+ .find(isFile);
+ if (target) visit(target);
+ }
+ };
+ visit(entry);
+ return [...seen];
+}
+
+/** Newest committer date (ISO 8601 / W3C Datetime) for every path Git has history for, keyed by top-level-relative path. */
+export function commitDates(git) {
+ const dates = new Map();
+ let date;
+ for (const line of git(['-c', 'core.quotePath=false', 'log', '--format=%x00%cI', '--name-only', '--no-renames', 'HEAD']).split('\n')) {
+ if (line.startsWith('\0')) date = line.slice(1);
+ else if (line && !dates.has(line)) dates.set(line, date);
+ }
+ return dates;
+}
+
+const newest = (dates) => dates.reduce((a, b) => (Date.parse(b) > Date.parse(a) ? b : a));
+
+/** A `route => lastmod` function for the site at `site`, read from the Git work tree that holds it. */
+export function sitemapLastmod({ site = SITE, git } = {}) {
+ git ??= (args) => execFileSync('git', ['-C', site, ...args], { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] }).trim();
+ const top = git(['rev-parse', '--show-toplevel']);
+ const head = git(['log', '-1', '--format=%cI', 'HEAD']);
+ const dates = commitDates(git);
+ const pagesRoot = join(site, 'src', 'pages');
+ const walk = (directory) => readdirSync(directory, { withFileTypes: true }).flatMap((entry) =>
+ entry.isDirectory() ? walk(join(directory, entry.name)) : entry.name.endsWith('.astro') ? [relative(pagesRoot, join(directory, entry.name))] : []);
+ const pages = walk(pagesRoot);
+ const closures = new Map();
+ const docs = docsDirectory(process.env, site);
+ const repoPath = (file) => relative(top, file).split(sep).join('/');
+ const chrome = new Set(SITE_CHROME);
+ const skip = (file) => chrome.has(repoPath(file));
+ const dateOf = (file) => dates.get(repoPath(file)) ?? head;
+ return (route) => {
+ const page = pageForRoute(route, pages);
+ if (!page) throw new Error(`No page source matches ${route}; cannot date it for the sitemap.`);
+ if (!closures.has(page)) closures.set(page, sourceClosure(join(pagesRoot, page), { skip }));
+ const files = [...closures.get(page)];
+ const doc = /^\/docs\/([^/]+)\/$/.exec(route)?.[1];
+ if (doc) files.push(join(docs, `${doc}.md`));
+ return newest(files.map(dateOf));
+ };
+}
diff --git a/site/scripts/sitemap-lastmod.test.ts b/site/scripts/sitemap-lastmod.test.ts
new file mode 100644
index 00000000..7efa82bb
--- /dev/null
+++ b/site/scripts/sitemap-lastmod.test.ts
@@ -0,0 +1,74 @@
+import { expect, test } from 'bun:test';
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { commitDates, pageForRoute, sitemapLastmod, sourceClosure } from './sitemap-lastmod.mjs';
+
+const pages = ['index.astro', '404.astro', 'docs/index.astro', 'docs/[slug].astro', 'gallery/[slug].astro', 'gallery/archive/index.astro', 'gallery/archive/[slug].astro', 'packs/troy/[...page].astro', 'scenes/farm.astro'];
+
+test('a route is matched to the page that emits it, static segments before parameters', () => {
+ expect(pageForRoute('/', pages)).toBe('index.astro');
+ expect(pageForRoute('/404.html', pages)).toBe('404.astro');
+ expect(pageForRoute('/docs/', pages)).toBe('docs/index.astro');
+ expect(pageForRoute('/docs/install/', pages)).toBe('docs/[slug].astro');
+ expect(pageForRoute('/gallery/archive/', pages)).toBe('gallery/archive/index.astro');
+ expect(pageForRoute('/gallery/archive/robot-arm/', pages)).toBe('gallery/archive/[slug].astro');
+ expect(pageForRoute('/gallery/barn/', pages)).toBe('gallery/[slug].astro');
+ expect(pageForRoute('/packs/troy/', pages)).toBe('packs/troy/[...page].astro');
+ expect(pageForRoute('/packs/troy/walls/', pages)).toBe('packs/troy/[...page].astro');
+ expect(pageForRoute('/scenes/farm/', pages)).toBe('scenes/farm.astro');
+ expect(pageForRoute('/unknown/deep/route/', pages)).toBeUndefined();
+});
+
+test('the source closure follows relative imports and skips stylesheets and packages', () => {
+ const files: Record<string, string> = {
+ '/s/pages/index.astro': "---\nimport Layout from '../layouts/Base.astro';\nimport data from '../data/home.json';\nimport { x } from '../lib/x';\nimport 'astro:content';\nimport '../styles/global.css';\nconst lazy = await import('../lib/lazy.mjs');\n---",
+ '/s/layouts/Base.astro': "import { y } from '../lib/x';",
+ '/s/data/home.json': '{"import": "../never.ts"}',
+ '/s/lib/x.ts': "export { z } from './z';",
+ '/s/lib/z/index.ts': 'export const z = 1;',
+ '/s/lib/lazy.mjs': '',
+ '/s/styles/global.css': '',
+ };
+ // resolve() makes Windows paths of these POSIX ones; compare them as POSIX paths.
+ const posix = (file: string) => file.replaceAll('\\', '/').replace(/^[A-Z]:/, '');
+ const read = (file: string) => files[posix(file)];
+ const isFile = (file: string) => posix(file) in files;
+ const closure = sourceClosure('/s/pages/index.astro', { read, isFile });
+ expect(closure.map(posix).sort()).toEqual(['/s/data/home.json', '/s/layouts/Base.astro', '/s/lib/lazy.mjs', '/s/lib/x.ts', '/s/lib/z/index.ts', '/s/pages/index.astro']);
+ // A skipped file is not dated, and neither is what only it imports.
+ const skip = (file: string) => posix(file) === '/s/lib/x.ts';
+ expect(sourceClosure('/s/pages/index.astro', { read, isFile, skip }).map(posix).sort()).toEqual(['/s/data/home.json', '/s/layouts/Base.astro', '/s/lib/lazy.mjs', '/s/pages/index.astro']);
+});
+
+test('commit dates keep the newest commit per path from git log order', () => {
+ const log = '\0' + '2026-10-25T22:07:37-04:00\n\nsite/a.json\ndocs/install.md\n\0' + '2026-09-01T10:00:00Z\n\nsite/a.json\nsite/b.ts\n';
+ const dates = commitDates(() => log);
+ expect(Object.fromEntries(dates)).toEqual({ 'site/a.json': '2026-10-25T22:07:37-04:00', 'docs/install.md': '2026-10-25T22:07:37-04:00', 'site/b.ts': '2026-09-01T10:00:00Z' });
+});
+
+test('a page is dated by its newest source but not the site chrome, a doc by its Markdown too, an uncommitted file at HEAD', () => {
+ const root = mkdtempSync(join(tmpdir(), 'kiln-lastmod-'));
+ try {
+ const site = join(root, 'site');
+ for (const directory of ['src/pages/docs', 'src/data', 'src/layouts', 'src/lib']) mkdirSync(join(site, directory), { recursive: true });
+ mkdirSync(join(root, 'docs'));
+ writeFileSync(join(site, 'src/pages/index.astro'), "import Base from '../layouts/BaseLayout.astro';\nimport { VERSION } from '../lib/config';\nimport home from '../data/home.json';");
+ writeFileSync(join(site, 'src/layouts/BaseLayout.astro'), '');
+ writeFileSync(join(site, 'src/lib/config.ts'), '');
+ writeFileSync(join(site, 'src/pages/docs/[slug].astro'), '');
+ writeFileSync(join(site, 'src/pages/new.astro'), '');
+ writeFileSync(join(site, 'src/data/home.json'), '{}');
+ writeFileSync(join(root, 'docs/install.md'), '# Install');
+ // The site chrome changed last; it redates no page.
+ const log = '\0' + '2026-10-23T16:19:55Z\n\nsite/src/layouts/BaseLayout.astro\nsite/src/lib/config.ts\n\0' + '2026-10-20T09:00:00+02:00\n\ndocs/install.md\n\0' + '2026-10-10T12:00:00Z\n\nsite/src/data/home.json\n\0' + '2026-08-01T12:00:00Z\n\nsite/src/pages/index.astro\nsite/src/pages/docs/[slug].astro\n';
+ const git = (args: string[]) => (args[0] === 'rev-parse' ? root : args[0] === 'log' && args[1] === '-1' ? '2026-10-24T00:00:00Z' : log);
+ const lastmod = sitemapLastmod({ site, git });
+ expect(lastmod('/')).toBe('2026-10-10T12:00:00Z');
+ expect(lastmod('/docs/install/')).toBe('2026-10-20T09:00:00+02:00');
+ expect(lastmod('/new/')).toBe('2026-10-24T00:00:00Z');
+ expect(() => lastmod('/missing/route/')).toThrow('No page source matches');
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
diff --git a/site/scripts/sitemap-routes.mjs b/site/scripts/sitemap-routes.mjs
index 02cdabdb..3341b692 100644
--- a/site/scripts/sitemap-routes.mjs
+++ b/site/scripts/sitemap-routes.mjs
@@ -9,3 +9,19 @@ export function compareSitemapRoutes(xml, expected) {
...[...actual].filter((path) => paths.indexOf(path) !== paths.lastIndexOf(path)).map((path) => `Duplicate sitemap route: ${path}`),
];
}
+
+// W3C Datetime as sitemaps.org specifies it: a date, or a date and time to at least minutes with a zone designator.
+const W3C_DATETIME = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2}))?$/;
+
+/** Every `<url>` carries exactly one `<lastmod>` that is a W3C Datetime, not in the future of `now`. */
+export function sitemapLastmodErrors(xml, now = Date.now()) {
+ const errors = [];
+ for (const [entry] of xml.matchAll(/<url>[\s\S]*?<\/url>/g)) {
+ const loc = /<loc>([^<]+)<\/loc>/.exec(entry)?.[1] ?? entry;
+ const dates = Array.from(entry.matchAll(/<lastmod>([^<]*)<\/lastmod>/g), (match) => match[1]);
+ if (dates.length !== 1) errors.push(`Sitemap URL has ${dates.length} lastmod values: ${loc}`);
+ else if (!W3C_DATETIME.test(dates[0]) || Number.isNaN(Date.parse(dates[0]))) errors.push(`Sitemap lastmod is not a W3C Datetime: ${loc} ${dates[0]}`);
+ else if (Date.parse(dates[0]) > now) errors.push(`Sitemap lastmod is in the future: ${loc} ${dates[0]}`);
+ }
+ return errors;
+}
diff --git a/site/scripts/sitemap-routes.test.ts b/site/scripts/sitemap-routes.test.ts
index 3376e33d..f666dac9 100644
--- a/site/scripts/sitemap-routes.test.ts
+++ b/site/scripts/sitemap-routes.test.ts
@@ -1,8 +1,18 @@
import { expect, test } from 'bun:test';
-import { compareSitemapRoutes } from './sitemap-routes.mjs';
+import { compareSitemapRoutes, sitemapLastmodErrors } from './sitemap-routes.mjs';
const xml = (paths: string[]) => `<urlset>${paths.map((path) => `<url><loc>https://kilnstudio.tools${path}</loc></url>`).join('')}</urlset>`;
test('served sitemap matches the exact local indexable route set', () => {
expect(compareSitemapRoutes(xml(['/','/docs/']), ['/docs/','/'])).toEqual([]);
expect(compareSitemapRoutes(xml(['/','/stale/']), ['/','/docs/'])).toEqual(['Missing sitemap route: /docs/', 'Unexpected sitemap route: /stale/']);
expect(compareSitemapRoutes(xml(['/','/']), ['/'])).toContain('Duplicate sitemap route: /');
});
+test('every sitemap URL carries one W3C Datetime lastmod that is not in the future', () => {
+ const now = Date.parse('2026-10-27T00:00:00Z');
+ const entry = (lastmod: string) => `<urlset><url><loc>https://kilnstudio.tools/</loc>${lastmod}</url></urlset>`;
+ expect(sitemapLastmodErrors(entry('<lastmod>2026-10-25T22:07:37-04:00</lastmod>'), now)).toEqual([]);
+ expect(sitemapLastmodErrors(entry('<lastmod>2026-10-25</lastmod>'), now)).toEqual([]);
+ expect(sitemapLastmodErrors(entry(''), now)).toEqual(['Sitemap URL has 0 lastmod values: https://kilnstudio.tools/']);
+ expect(sitemapLastmodErrors(entry('<lastmod>Oct 25 2026</lastmod>'), now)).toEqual(['Sitemap lastmod is not a W3C Datetime: https://kilnstudio.tools/ Oct 25 2026']);
+ expect(sitemapLastmodErrors(entry('<lastmod>2026-10-25T22:07:37</lastmod>'), now)).toHaveLength(1);
+ expect(sitemapLastmodErrors(entry('<lastmod>2027-01-01</lastmod>'), now)).toEqual(['Sitemap lastmod is in the future: https://kilnstudio.tools/ 2027-01-01']);
+});
diff --git a/site/scripts/static-validation-core.mjs b/site/scripts/static-validation-core.mjs
index 39b2d24a..7911efc6 100644
--- a/site/scripts/static-validation-core.mjs
+++ b/site/scripts/static-validation-core.mjs
@@ -336,6 +336,12 @@ export function parseHeaderRules(text) {
const line = raw.trim();
if (!line || line.startsWith('#')) continue;
if (line.startsWith('/')) rules.push({ path: line, set: {}, unset: [] });
+ // An absolute rule names its host too, as Pages allows ("https://" only, no port); `:name` matches one host label.
+ else if (line.startsWith('https://')) {
+ const url = /^https:\/\/([^/]+)(\/.*)$/.exec(line);
+ if (!url || /:\d*$/.test(url[1])) throw new Error(`_headers: an absolute rule needs an https host without a port and a path: ${line}`);
+ rules.push({ host: url[1], path: url[2], set: {}, unset: [] });
+ }
else if (!rules.length) throw new Error(`_headers: a header before any path: ${line}`);
else if (line.startsWith('! ')) rules.at(-1).unset.push(line.slice(2).trim().toLowerCase());
else {
@@ -353,11 +359,24 @@ const headerRuleRegExp = (path) => {
return new RegExp(`^${pattern.replace(/:([A-Za-z]\w*)/g, '(?<$1>[^/]+)')}$`);
};
-/** The headers served for a path: matching rules in order; `!` removes, a name set twice is joined with a comma. */
-export function headersFor(rules, pathname) {
+/** A rule host as the host matches it: `:name` is one label, since a host placeholder ends at a period. */
+const headerHostRegExp = (host) => {
+ const labels = host.split('.').map((label) => (/^:[A-Za-z]\w*$/.test(label) ? '[^.]+' : label.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&')));
+ return new RegExp(`^${labels.join('\\.')}$`, 'i');
+};
+
+/** The Pages project's own hosts, which must stay unindexed: its production alias and a deployment and a branch alias. */
+export const PAGES_DEV_HOSTS = ['kilnstudio.pages.dev', '0123abcd.kilnstudio.pages.dev', 'production.kilnstudio.pages.dev'];
+
+/**
+ * The headers served for a path on `host` (the production host unless named): matching rules in order; `!` removes, a
+ * name set twice is joined with a comma. Path rules apply on every host; an absolute rule only on the hosts it names.
+ */
+export function headersFor(rules, pathname, host = new URL(ORIGIN).host) {
const headers = new Map();
const seen = new Set();
for (const rule of rules) {
+ if (rule.host && !headerHostRegExp(rule.host).test(host)) continue;
if (!headerRuleRegExp(rule.path).test(pathname)) continue;
for (const name of rule.unset) headers.delete(name);
for (const [name, value] of Object.entries(rule.set)) {
@@ -398,6 +417,13 @@ export function headersErrors({ text, inlineScripts = [], files = [], sealedFile
if (page['content-security-policy']) errors.push('The content security policy is enforced; it stays report-only until the owner decides');
const policy = page['content-security-policy-report-only'];
if (!policy) errors.push('Pages carry no Content-Security-Policy-Report-Only');
+ // The Pages project also answers on its pages.dev hosts with the same bytes; only the production host is indexed.
+ if (/noindex/i.test(page['x-robots-tag'] ?? '')) errors.push(`The production host ${new URL(ORIGIN).host} must not be served X-Robots-Tag: noindex`);
+ for (const host of PAGES_DEV_HOSTS) {
+ for (const path of ['/', '/docs/', '/robots.txt']) {
+ if (!/noindex/i.test(headersFor(rules, path, host)['x-robots-tag'] ?? '')) errors.push(`https://${host}${path} must be served X-Robots-Tag: noindex`);
+ }
+ }
const cache = page['cache-control'] ?? '';
const maxAge = Number(cache.match(/max-age=(\d+)/)?.[1] ?? Number.NaN);
if (!(maxAge <= 300) || /immutable/.test(cache)) errors.push(`HTML must be cached briefly and revalidated, got Cache-Control ${JSON.stringify(cache || null)}`);
diff --git a/site/scripts/static-validation.test.ts b/site/scripts/static-validation.test.ts
index ab02e216..5fa835f1 100644
--- a/site/scripts/static-validation.test.ts
+++ b/site/scripts/static-validation.test.ts
@@ -379,12 +379,18 @@ describe('the header layer in _headers (engineering review, finding 11)', () =>
/scene-runtime/:scene/*.js
! Cache-Control
Cache-Control: public, max-age=31536000, immutable
+
+https://kilnstudio.pages.dev/*
+ X-Robots-Tag: noindex
+
+https://:version.kilnstudio.pages.dev/*
+ X-Robots-Tag: noindex
${extra}`;
const good = file(`default-src 'self'; script-src 'self' ${scriptHashSource(script)}`);
test('rules apply in order, "!" removes what an earlier rule set, placeholders match one segment', () => {
const rules = parseHeaderRules(good);
- expect(rules.map((rule) => rule.path)).toEqual(['/*', '/_astro/*', '/scene-runtime/:scene/*.js']);
+ expect(rules.map((rule) => `${rule.host ?? ''}${rule.path}`)).toEqual(['/*', '/_astro/*', '/scene-runtime/:scene/*.js', 'kilnstudio.pages.dev/*', ':version.kilnstudio.pages.dev/*']);
expect(headersFor(rules, '/')['cache-control']).toBe('public, max-age=0, must-revalidate');
expect(headersFor(rules, '/_astro/viewer.GoKKfp1n.js')['cache-control']).toBe('public, max-age=31536000, immutable');
expect(headersFor(rules, '/scene-runtime/golden-gate/index-CA13LcNQ.js')['cache-control']).toBe('public, max-age=31536000, immutable');
@@ -424,6 +430,26 @@ ${extra}`;
expect(headersErrors({ text: good.replace('max-age=0, must-revalidate', 'max-age=86400'), inlineScripts: [script] })).toEqual(['HTML must be cached briefly and revalidated, got Cache-Control "public, max-age=86400"']);
});
+ test('absolute rules apply only on the hosts they name: the pages.dev hosts are noindex, the production host is not', () => {
+ const rules = parseHeaderRules(good);
+ expect(headersFor(rules, '/docs/')['x-robots-tag']).toBeUndefined();
+ expect(headersFor(rules, '/docs/', 'kilnstudio.tools')['x-robots-tag']).toBeUndefined();
+ expect(headersFor(rules, '/docs/', 'kilnstudio.pages.dev')['x-robots-tag']).toBe('noindex');
+ expect(headersFor(rules, '/', 'a1b2c3d4.kilnstudio.pages.dev')['x-robots-tag']).toBe('noindex');
+ expect(headersFor(rules, '/', 'pr-7.kilnstudio.pages.dev')['x-robots-tag']).toBe('noindex');
+ expect(headersFor(rules, '/', 'a.b.kilnstudio.pages.dev')['x-robots-tag']).toBeUndefined();
+ expect(headersFor(rules, '/', 'other.pages.dev')['x-robots-tag']).toBeUndefined();
+ expect(headersFor(rules, '/docs/', 'kilnstudio.pages.dev')['x-content-type-options']).toBe('nosniff');
+ expect(() => parseHeaderRules('https://kilnstudio.pages.dev:8443/*\n X-Robots-Tag: noindex')).toThrow('without a port');
+ });
+
+ test('a pages.dev host left indexable, or a noindex production host, is reported', () => {
+ const open = good.replace('\nhttps://kilnstudio.pages.dev/*\n X-Robots-Tag: noindex\n', '\n');
+ expect(headersErrors({ text: open, inlineScripts: [script] })).toEqual(['/', '/docs/', '/robots.txt'].map((path) => `https://kilnstudio.pages.dev${path} must be served X-Robots-Tag: noindex`));
+ const everywhere = `${good}\n/*\n X-Robots-Tag: noindex\n`;
+ expect(headersErrors({ text: everywhere, inlineScripts: [script] })).toEqual(['The production host kilnstudio.tools must not be served X-Robots-Tag: noindex']);
+ });
+
test('inline scripts are found in pages; JSON-LD data blocks are not scripts to a CSP', () => {
const page = inspectHtml(`<html><head><script>${script}</script><script type="application/ld+json">{"@type":"Thing"}</script><script type="module" src="/_astro/a.js"></script></head><body></body></html>`);
expect(page.inlineScripts).toEqual([script]);
diff --git a/site/scripts/validate-static.mjs b/site/scripts/validate-static.mjs
index efb5b7fb..5558c8a6 100644
--- a/site/scripts/validate-static.mjs
+++ b/site/scripts/validate-static.mjs
@@ -9,6 +9,7 @@ import {sealedTroyCache} from './sealed-cache.mjs';
import {viewerModelInput} from './viewer-model-input.mjs';
import { glbJsonText } from './private-data.mjs';
import {validatePublicText} from './validate-public-text.mjs';
+import { sitemapLastmodErrors } from './sitemap-routes.mjs';
const site = resolve(dirname(fileURLToPath(import.meta.url)), '..');
const args = process.argv.slice(2);
@@ -130,6 +131,7 @@ let sitemapUrls = [];
try {
const sitemap = await readFile(resolve(root, 'sitemap.xml'), 'utf8');
sitemapUrls = Array.from(sitemap.matchAll(/<loc>(.*?)<\/loc>/g), (match) => match[1]);
+ for (const error of sitemapLastmodErrors(sitemap)) add('/sitemap.xml', 'sitemap', error);
for (const [route, page] of pages) {
const url = new URL(route, ORIGIN).href;
if (page.noindex && sitemapUrls.includes(url)) add(route, 'sitemap', 'noindex page is in the sitemap');