Build and revise procedural 3D assets with your coding agent. Local MCP server, CLI and TypeScript engine with rendered review, editable source and GLB export.
JavaScript 3d3d-assets3d-modelingagent-skillsagent-toolsai-agentsasset-pipelinecligame-assetsglbgltfmcpmcp-servermodel-context-protocolprocedural-generationprocedural-modelingtext-to-3dthreejstypescript
feat(site): date sitemap URLs and keep pages.dev hosts out of search (#278)
Matthew Kissinger6ff8f8ab89cd
diff --git a/docs/site-indexing.md b/docs/site-indexing.mdindex b7df527d..1660a944 100644--- a/docs/site-indexing.md+++ b/docs/site-indexing.md@@ -60,3 +60,23 @@ returned 200. The existing sitemap index was resubmitted in Search Console, whicconfirmed **Sitemap submitted successfully** and then **Success**, with its lastread on September 6. Discovered pages initially remained zero; processing andindexing are separate from successful retrieval. No account permissions changed.++## Sitemap dates and pages.dev hosts — October 26, 2026++Live checks against build `238d1aa8` (the `main` head): all 329 sitemap URLs returned+200 with a self-referencing canonical on `https://kilnstudio.tools` and no `noindex`;+every share image returned 200 at 1200×630. The sitemap carried no `<lastmod>`, and+`kilnstudio.pages.dev` served the full site with no `X-Robots-Tag`. Preview aliases+already answered with `X-Robots-Tag: noindex`.++- `finalize-site` now writes a `<lastmod>` for every sitemap URL+ (`site/scripts/sitemap-lastmod.mjs`): the newest commit date among the files the+ page is built from (its Astro page, the local modules and data records it imports,+ and for a guide its Markdown). The shared base layout and site constants are not+ followed, so a navigation or version-label change does not redate every page.+ `validate-static` fails a sitemap URL without exactly one W3C Datetime `<lastmod>`.+- `_headers` sends `X-Robots-Tag: noindex` on `kilnstudio.pages.dev` and every+ `*.kilnstudio.pages.dev` alias, following Cloudflare's documented absolute-URL rule.+ The production host is unaffected; `validate-static` checks both.++Robots rules, canonicals and the noindex pages are unchanged.diff --git a/site/public/_headers b/site/public/_headersindex 5282cec9..425ea5db 100644--- a/site/public/_headers+++ b/site/public/_headers@@ -17,6 +17,15 @@Cache-Control: public, max-age=0, must-revalidateContent-Security-Policy-Report-Only: default-src 'self'; script-src 'self' 'wasm-unsafe-eval' 'sha256-wAGukfde07th8FeXAlx31NqovrboToJ/niuiFbyoI70=' 'sha256-nQyK+pVXDH5RpnnZ8bk4KjsgiC91NvDr20Qc7mo8J4w=' 'sha256-qsHLNpEk33lPgo1MnVDl/NNWHbeSVPK4+ksw8A84/ls=' 'sha256-pi2avave5D7tqBWNSP54Cy8gsoNF+VSK2JuekU4lFcg=' 'sha256-ynl5VnLFfq74TN1iDcyssM5zkT3yce8NSUQIazX5/GY='; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; connect-src 'self' blob: https://assets.kilnstudio.tools; worker-src blob:+# The project's own pages.dev hosts serve these same bytes (production at kilnstudio.pages.dev, each deployment and+# branch alias beneath it), so they are kept out of search indexes; only https://kilnstudio.tools is indexed. Pages+# matches an absolute rule on its host, where ":name" is one label (developers.cloudflare.com/pages/configuration/headers/).+https://kilnstudio.pages.dev/*+ X-Robots-Tag: noindex++https://:version.kilnstudio.pages.dev/*+ X-Robots-Tag: noindex+# Build output named by content hash: a changed file gets a new name, so these never need revalidating./_astro/*! Cache-Controldiff --git a/site/scripts/finalize-site.mjs b/site/scripts/finalize-site.mjsindex 744c3c57..ffd204d3 100644--- a/site/scripts/finalize-site.mjs+++ b/site/scripts/finalize-site.mjs@@ -3,6 +3,7 @@ import { dirname, relative, resolve } from 'node:path';import { fileURLToPath } from 'node:url';import { ARCHIVE_INDEX_ROUTE, inspectHtml, isArchiveItemRoute, ORIGIN, routeForFile, scriptHashSource } from './static-validation-core.mjs';import {sealedTroyCache} from './sealed-cache.mjs';+import { sitemapLastmod } from './sitemap-lastmod.mjs';const escapeXml = (text) => text.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>');const link = (page) => `- [${page.title.replace(/ · Archive · Kiln$| · Kiln docs$| · Kiln$| — Kiln$| — source and 3D model \| Kiln$| — Kiln Commons$/g, '')}](${new URL(page.route, ORIGIN).href}): ${page.description ?? ''}`;@@ -22,8 +23,8 @@ export function buildHeaders(template, pages) {return text;}-/** Derive discovery files from the pages that were actually emitted by Astro. */-export function buildIndexFiles(pages) {+/** Derive discovery files from the pages that were actually emitted by Astro. `lastmod(route)` dates each sitemap URL. */+export function buildIndexFiles(pages, { lastmod } = {}) {// Archive item pages are excluded by route as well as by their noindex marker: the marker is the source of truth,// and validate-static fails a build in which an item lost it, but the discovery files never depend on that alone.const published = pages.filter((page) => !page.noindex && page.route !== '/404.html' && !isArchiveItemRoute(page.route)).sort((a, b) => a.route.localeCompare(b.route));@@ -31,7 +32,10 @@ export function buildIndexFiles(pages) {const docs = published.filter((page) => page.route.startsWith('/docs/'));const catalog = published.filter((page) => /^\/(?:packs|gallery|scenes)\/(?:[^/]+\/)?$/.test(page.route) && page.route !== ARCHIVE_INDEX_ROUTE);const archive = published.find((page) => page.route === ARCHIVE_INDEX_ROUTE);- const sitemap = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${published.map((page) => ` <url><loc>${escapeXml(new URL(page.route, ORIGIN).href)}</loc></url>`).join('\n')}\n</urlset>\n`;+ const sitemap = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${published.map((page) => {+ const date = lastmod?.(page.route);+ return ` <url><loc>${escapeXml(new URL(page.route, ORIGIN).href)}</loc>${date ? `<lastmod>${escapeXml(date)}</lastmod>` : ''}</url>`;+ }).join('\n')}\n</urlset>\n`;// The owner's Search Console holds a submission of /sitemap-index.xml from the previous site (docs/site-indexing.md), so the path// keeps answering with an index of the one sitemap. It is generated with the sitemap, never a checked-in copy that could drift.const sitemapIndex = `<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n <sitemap><loc>${ORIGIN}/sitemap.xml</loc></sitemap>\n</sitemapindex>\n`;@@ -70,7 +74,7 @@ export async function finalizeSite({ dist = resolve(dirname(fileURLToPath(importpages.push({ route: routeForFile(relative(dist, file)), ...inspectHtml(await readFile(file, 'utf8')) });}if (!pages.some((page) => page.route === '/')) throw new Error(`Astro home output missing in ${dist}; build the site before finalizing.`);- const { sitemap, sitemapIndex, robots, llms } = buildIndexFiles(pages);+ const { sitemap, sitemapIndex, robots, llms } = buildIndexFiles(pages, { lastmod: sitemapLastmod() });const headerPath=resolve(dist,'_headers');const sealedCache=await sealedTroyCache(dist);const headers=buildHeaders(await readFile(headerPath,'utf8'),pages)+sealedCache.headers;diff --git a/site/scripts/finalize-site.test.ts b/site/scripts/finalize-site.test.tsindex 9f6b0f85..69821a15 100644--- a/site/scripts/finalize-site.test.ts+++ b/site/scripts/finalize-site.test.ts@@ -88,3 +88,11 @@ test('a sitemap index lists the sitemap and only the sitemap', () => {expect(sitemapIndex).toBe('<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n <sitemap><loc>https://kilnstudio.tools/sitemap.xml</loc></sitemap>\n</sitemapindex>\n');expect(locs(sitemapIndex)).toEqual(['https://kilnstudio.tools/sitemap.xml']);});++test('each sitemap URL carries the lastmod its route is dated with', () => {+ const lastmod = (route: string) => (route === '/' ? '2026-10-25T22:07:37-04:00' : '2026-09-01T10:00:00Z');+ const { sitemap } = buildIndexFiles(pages, { lastmod });+ expect(sitemap).toContain('<url><loc>https://kilnstudio.tools/</loc><lastmod>2026-10-25T22:07:37-04:00</lastmod></url>');+ expect(sitemap).toContain('<url><loc>https://kilnstudio.tools/docs/install/</loc><lastmod>2026-09-01T10:00:00Z</lastmod></url>');+ expect(locs(sitemap).length).toBe((sitemap.match(/<lastmod>/g) ?? []).length);+});diff --git a/site/scripts/sitemap-lastmod.mjs b/site/scripts/sitemap-lastmod.mjsnew file mode 100644index 00000000..7dc67873--- /dev/null+++ b/site/scripts/sitemap-lastmod.mjs@@ -0,0 +1,117 @@+/**+ * `<lastmod>` for each sitemap URL: the committer date of the newest commit that touched a file the page is built from.+ *+ * Google uses lastmod only when it is "consistently and verifiably accurate"+ * (https://developers.google.com/search/docs/crawling-indexing/sitemaps/build-sitemap), so the date is derived, never+ * typed: a route is matched to its Astro page file, the page's local imports are followed (data records, components,+ * layouts, libraries; stylesheets are skipped because a CSS change is not a content change), and a documentation route+ * adds its repository Markdown (from `KILN_SITE_DOCS_DIR`, as the docs loader reads it). The newest Git date among+ * those files is the page's lastmod. A file with no commit yet (an uncommitted addition) counts as changed at HEAD.+ * The shared chrome every page carries (`SITE_CHROME`: the base layout's header, footer and share card, and the+ * site constants with the release label) is not followed: Google asks for the last *significant* change to a page, and+ * a navigation link or version label changing on all pages at once would otherwise redate the whole sitemap.+ * Files read at build time from untracked inputs (the restored example archive, generated sources) are pinned by+ * tracked records, which are the files that carry the date.+ */+import { execFileSync } from 'node:child_process';+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';+import { dirname, join, relative, resolve, sep } from 'node:path';+import { fileURLToPath } from 'node:url';+import { docsDirectory } from './source-dirs.mjs';++const SITE = resolve(dirname(fileURLToPath(import.meta.url)), '..');+const EXTENSIONS = ['', '.ts', '.tsx', '.mjs', '.js', '.astro', '.json'];+const SOURCE = /\.(?:astro|[cm]?[jt]sx?|json)$/;+/** Repository paths of the site-wide chrome that no page's lastmod follows (see above). */+export const SITE_CHROME = ['site/src/layouts/BaseLayout.astro', 'site/src/lib/config.ts'];+const IMPORT = /(?:^|[\s;])(?:import|export)\s+(?:[^'"`;]*?\sfrom\s*)?['"]([^'"]+)['"]|\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)/g;++/** The Astro page pattern for a file under src/pages: static segments, `[name]` (one segment) or `[...name]` (any). */+export function pagePattern(file) {+ const path = file.replaceAll('\\', '/').replace(/\.astro$/, '').replace(/(?:^|\/)index$/, '');+ return path ? path.split('/') : [];+}++/** The page file that emitted `route`, by Astro's priority: more static segments first, then single params over rest. */+export function pageForRoute(route, files) {+ const segments = route.replace(/\.html$/, '').split('/').filter(Boolean);+ const matches = (pattern, at = 0, index = 0) => {+ if (index === pattern.length) return at === segments.length;+ const part = pattern[index];+ if (/^\[\.\.\.[^\]]+\]$/.test(part)) {+ for (let end = segments.length; end >= at; end--) if (matches(pattern, end, index + 1)) return true;+ return false;+ }+ if (at >= segments.length) return false;+ return (/^\[[^\]]+\]$/.test(part) || part === segments[at]) && matches(pattern, at + 1, index + 1);+ };+ const score = (pattern) => [pattern.filter((part) => !part.startsWith('[')).length, -pattern.filter((part) => part.startsWith('[...')).length];+ let best;+ for (const file of files) {+ const pattern = pagePattern(file);+ if (!matches(pattern)) continue;+ const [fixed, rest] = score(pattern);+ if (!best || fixed > best.fixed || (fixed === best.fixed && rest > best.rest)) best = { file, fixed, rest };+ }+ return best?.file;+}++/** Every local file reachable from `entry` through relative imports, `entry` included, except files `skip` refuses. */+export function sourceClosure(entry, { read = (file) => readFileSync(file, 'utf8'), isFile = (file) => existsSync(file) && statSync(file).isFile(), skip = () => false } = {}) {+ const seen = new Set();+ const visit = (file) => {+ if (seen.has(file) || (file !== entry && skip(file))) return;+ seen.add(file);+ if (!SOURCE.test(file) || file.endsWith('.json')) return;+ for (const match of read(file).matchAll(IMPORT)) {+ const specifier = (match[1] ?? match[2]).split('?')[0];+ if (!specifier.startsWith('.') || specifier.endsWith('.css')) continue;+ const base = resolve(dirname(file), specifier);+ const target = [...EXTENSIONS.map((extension) => base + extension), ...EXTENSIONS.slice(1).map((extension) => join(base, `index${extension}`))]+ .find(isFile);+ if (target) visit(target);+ }+ };+ visit(entry);+ return [...seen];+}++/** Newest committer date (ISO 8601 / W3C Datetime) for every path Git has history for, keyed by top-level-relative path. */+export function commitDates(git) {+ const dates = new Map();+ let date;+ for (const line of git(['-c', 'core.quotePath=false', 'log', '--format=%x00%cI', '--name-only', '--no-renames', 'HEAD']).split('\n')) {+ if (line.startsWith('\0')) date = line.slice(1);+ else if (line && !dates.has(line)) dates.set(line, date);+ }+ return dates;+}++const newest = (dates) => dates.reduce((a, b) => (Date.parse(b) > Date.parse(a) ? b : a));++/** A `route => lastmod` function for the site at `site`, read from the Git work tree that holds it. */+export function sitemapLastmod({ site = SITE, git } = {}) {+ git ??= (args) => execFileSync('git', ['-C', site, ...args], { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] }).trim();+ const top = git(['rev-parse', '--show-toplevel']);+ const head = git(['log', '-1', '--format=%cI', 'HEAD']);+ const dates = commitDates(git);+ const pagesRoot = join(site, 'src', 'pages');+ const walk = (directory) => readdirSync(directory, { withFileTypes: true }).flatMap((entry) =>+ entry.isDirectory() ? walk(join(directory, entry.name)) : entry.name.endsWith('.astro') ? [relative(pagesRoot, join(directory, entry.name))] : []);+ const pages = walk(pagesRoot);+ const closures = new Map();+ const docs = docsDirectory(process.env, site);+ const repoPath = (file) => relative(top, file).split(sep).join('/');+ const chrome = new Set(SITE_CHROME);+ const skip = (file) => chrome.has(repoPath(file));+ const dateOf = (file) => dates.get(repoPath(file)) ?? head;+ return (route) => {+ const page = pageForRoute(route, pages);+ if (!page) throw new Error(`No page source matches ${route}; cannot date it for the sitemap.`);+ if (!closures.has(page)) closures.set(page, sourceClosure(join(pagesRoot, page), { skip }));+ const files = [...closures.get(page)];+ const doc = /^\/docs\/([^/]+)\/$/.exec(route)?.[1];+ if (doc) files.push(join(docs, `${doc}.md`));+ return newest(files.map(dateOf));+ };+}diff --git a/site/scripts/sitemap-lastmod.test.ts b/site/scripts/sitemap-lastmod.test.tsnew file mode 100644index 00000000..7efa82bb--- /dev/null+++ b/site/scripts/sitemap-lastmod.test.ts@@ -0,0 +1,74 @@+import { expect, test } from 'bun:test';+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { commitDates, pageForRoute, sitemapLastmod, sourceClosure } from './sitemap-lastmod.mjs';++const pages = ['index.astro', '404.astro', 'docs/index.astro', 'docs/[slug].astro', 'gallery/[slug].astro', 'gallery/archive/index.astro', 'gallery/archive/[slug].astro', 'packs/troy/[...page].astro', 'scenes/farm.astro'];++test('a route is matched to the page that emits it, static segments before parameters', () => {+ expect(pageForRoute('/', pages)).toBe('index.astro');+ expect(pageForRoute('/404.html', pages)).toBe('404.astro');+ expect(pageForRoute('/docs/', pages)).toBe('docs/index.astro');+ expect(pageForRoute('/docs/install/', pages)).toBe('docs/[slug].astro');+ expect(pageForRoute('/gallery/archive/', pages)).toBe('gallery/archive/index.astro');+ expect(pageForRoute('/gallery/archive/robot-arm/', pages)).toBe('gallery/archive/[slug].astro');+ expect(pageForRoute('/gallery/barn/', pages)).toBe('gallery/[slug].astro');+ expect(pageForRoute('/packs/troy/', pages)).toBe('packs/troy/[...page].astro');+ expect(pageForRoute('/packs/troy/walls/', pages)).toBe('packs/troy/[...page].astro');+ expect(pageForRoute('/scenes/farm/', pages)).toBe('scenes/farm.astro');+ expect(pageForRoute('/unknown/deep/route/', pages)).toBeUndefined();+});++test('the source closure follows relative imports and skips stylesheets and packages', () => {+ const files: Record<string, string> = {+ '/s/pages/index.astro': "---\nimport Layout from '../layouts/Base.astro';\nimport data from '../data/home.json';\nimport { x } from '../lib/x';\nimport 'astro:content';\nimport '../styles/global.css';\nconst lazy = await import('../lib/lazy.mjs');\n---",+ '/s/layouts/Base.astro': "import { y } from '../lib/x';",+ '/s/data/home.json': '{"import": "../never.ts"}',+ '/s/lib/x.ts': "export { z } from './z';",+ '/s/lib/z/index.ts': 'export const z = 1;',+ '/s/lib/lazy.mjs': '',+ '/s/styles/global.css': '',+ };+ // resolve() makes Windows paths of these POSIX ones; compare them as POSIX paths.+ const posix = (file: string) => file.replaceAll('\\', '/').replace(/^[A-Z]:/, '');+ const read = (file: string) => files[posix(file)];+ const isFile = (file: string) => posix(file) in files;+ const closure = sourceClosure('/s/pages/index.astro', { read, isFile });+ expect(closure.map(posix).sort()).toEqual(['/s/data/home.json', '/s/layouts/Base.astro', '/s/lib/lazy.mjs', '/s/lib/x.ts', '/s/lib/z/index.ts', '/s/pages/index.astro']);+ // A skipped file is not dated, and neither is what only it imports.+ const skip = (file: string) => posix(file) === '/s/lib/x.ts';+ expect(sourceClosure('/s/pages/index.astro', { read, isFile, skip }).map(posix).sort()).toEqual(['/s/data/home.json', '/s/layouts/Base.astro', '/s/lib/lazy.mjs', '/s/pages/index.astro']);+});++test('commit dates keep the newest commit per path from git log order', () => {+ const log = '\0' + '2026-10-25T22:07:37-04:00\n\nsite/a.json\ndocs/install.md\n\0' + '2026-09-01T10:00:00Z\n\nsite/a.json\nsite/b.ts\n';+ const dates = commitDates(() => log);+ expect(Object.fromEntries(dates)).toEqual({ 'site/a.json': '2026-10-25T22:07:37-04:00', 'docs/install.md': '2026-10-25T22:07:37-04:00', 'site/b.ts': '2026-09-01T10:00:00Z' });+});++test('a page is dated by its newest source but not the site chrome, a doc by its Markdown too, an uncommitted file at HEAD', () => {+ const root = mkdtempSync(join(tmpdir(), 'kiln-lastmod-'));+ try {+ const site = join(root, 'site');+ for (const directory of ['src/pages/docs', 'src/data', 'src/layouts', 'src/lib']) mkdirSync(join(site, directory), { recursive: true });+ mkdirSync(join(root, 'docs'));+ writeFileSync(join(site, 'src/pages/index.astro'), "import Base from '../layouts/BaseLayout.astro';\nimport { VERSION } from '../lib/config';\nimport home from '../data/home.json';");+ writeFileSync(join(site, 'src/layouts/BaseLayout.astro'), '');+ writeFileSync(join(site, 'src/lib/config.ts'), '');+ writeFileSync(join(site, 'src/pages/docs/[slug].astro'), '');+ writeFileSync(join(site, 'src/pages/new.astro'), '');+ writeFileSync(join(site, 'src/data/home.json'), '{}');+ writeFileSync(join(root, 'docs/install.md'), '# Install');+ // The site chrome changed last; it redates no page.+ const log = '\0' + '2026-10-23T16:19:55Z\n\nsite/src/layouts/BaseLayout.astro\nsite/src/lib/config.ts\n\0' + '2026-10-20T09:00:00+02:00\n\ndocs/install.md\n\0' + '2026-10-10T12:00:00Z\n\nsite/src/data/home.json\n\0' + '2026-08-01T12:00:00Z\n\nsite/src/pages/index.astro\nsite/src/pages/docs/[slug].astro\n';+ const git = (args: string[]) => (args[0] === 'rev-parse' ? root : args[0] === 'log' && args[1] === '-1' ? '2026-10-24T00:00:00Z' : log);+ const lastmod = sitemapLastmod({ site, git });+ expect(lastmod('/')).toBe('2026-10-10T12:00:00Z');+ expect(lastmod('/docs/install/')).toBe('2026-10-20T09:00:00+02:00');+ expect(lastmod('/new/')).toBe('2026-10-24T00:00:00Z');+ expect(() => lastmod('/missing/route/')).toThrow('No page source matches');+ } finally {+ rmSync(root, { recursive: true, force: true });+ }+});diff --git a/site/scripts/sitemap-routes.mjs b/site/scripts/sitemap-routes.mjsindex 02cdabdb..3341b692 100644--- a/site/scripts/sitemap-routes.mjs+++ b/site/scripts/sitemap-routes.mjs@@ -9,3 +9,19 @@ export function compareSitemapRoutes(xml, expected) {...[...actual].filter((path) => paths.indexOf(path) !== paths.lastIndexOf(path)).map((path) => `Duplicate sitemap route: ${path}`),];}++// W3C Datetime as sitemaps.org specifies it: a date, or a date and time to at least minutes with a zone designator.+const W3C_DATETIME = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2}))?$/;++/** Every `<url>` carries exactly one `<lastmod>` that is a W3C Datetime, not in the future of `now`. */+export function sitemapLastmodErrors(xml, now = Date.now()) {+ const errors = [];+ for (const [entry] of xml.matchAll(/<url>[\s\S]*?<\/url>/g)) {+ const loc = /<loc>([^<]+)<\/loc>/.exec(entry)?.[1] ?? entry;+ const dates = Array.from(entry.matchAll(/<lastmod>([^<]*)<\/lastmod>/g), (match) => match[1]);+ if (dates.length !== 1) errors.push(`Sitemap URL has ${dates.length} lastmod values: ${loc}`);+ else if (!W3C_DATETIME.test(dates[0]) || Number.isNaN(Date.parse(dates[0]))) errors.push(`Sitemap lastmod is not a W3C Datetime: ${loc} ${dates[0]}`);+ else if (Date.parse(dates[0]) > now) errors.push(`Sitemap lastmod is in the future: ${loc} ${dates[0]}`);+ }+ return errors;+}diff --git a/site/scripts/sitemap-routes.test.ts b/site/scripts/sitemap-routes.test.tsindex 3376e33d..f666dac9 100644--- a/site/scripts/sitemap-routes.test.ts+++ b/site/scripts/sitemap-routes.test.ts@@ -1,8 +1,18 @@import { expect, test } from 'bun:test';-import { compareSitemapRoutes } from './sitemap-routes.mjs';+import { compareSitemapRoutes, sitemapLastmodErrors } from './sitemap-routes.mjs';const xml = (paths: string[]) => `<urlset>${paths.map((path) => `<url><loc>https://kilnstudio.tools${path}</loc></url>`).join('')}</urlset>`;test('served sitemap matches the exact local indexable route set', () => {expect(compareSitemapRoutes(xml(['/','/docs/']), ['/docs/','/'])).toEqual([]);expect(compareSitemapRoutes(xml(['/','/stale/']), ['/','/docs/'])).toEqual(['Missing sitemap route: /docs/', 'Unexpected sitemap route: /stale/']);expect(compareSitemapRoutes(xml(['/','/']), ['/'])).toContain('Duplicate sitemap route: /');});+test('every sitemap URL carries one W3C Datetime lastmod that is not in the future', () => {+ const now = Date.parse('2026-10-27T00:00:00Z');+ const entry = (lastmod: string) => `<urlset><url><loc>https://kilnstudio.tools/</loc>${lastmod}</url></urlset>`;+ expect(sitemapLastmodErrors(entry('<lastmod>2026-10-25T22:07:37-04:00</lastmod>'), now)).toEqual([]);+ expect(sitemapLastmodErrors(entry('<lastmod>2026-10-25</lastmod>'), now)).toEqual([]);+ expect(sitemapLastmodErrors(entry(''), now)).toEqual(['Sitemap URL has 0 lastmod values: https://kilnstudio.tools/']);+ expect(sitemapLastmodErrors(entry('<lastmod>Oct 25 2026</lastmod>'), now)).toEqual(['Sitemap lastmod is not a W3C Datetime: https://kilnstudio.tools/ Oct 25 2026']);+ expect(sitemapLastmodErrors(entry('<lastmod>2026-10-25T22:07:37</lastmod>'), now)).toHaveLength(1);+ expect(sitemapLastmodErrors(entry('<lastmod>2027-01-01</lastmod>'), now)).toEqual(['Sitemap lastmod is in the future: https://kilnstudio.tools/ 2027-01-01']);+});diff --git a/site/scripts/static-validation-core.mjs b/site/scripts/static-validation-core.mjsindex 39b2d24a..7911efc6 100644--- a/site/scripts/static-validation-core.mjs+++ b/site/scripts/static-validation-core.mjs@@ -336,6 +336,12 @@ export function parseHeaderRules(text) {const line = raw.trim();if (!line || line.startsWith('#')) continue;if (line.startsWith('/')) rules.push({ path: line, set: {}, unset: [] });+ // An absolute rule names its host too, as Pages allows ("https://" only, no port); `:name` matches one host label.+ else if (line.startsWith('https://')) {+ const url = /^https:\/\/([^/]+)(\/.*)$/.exec(line);+ if (!url || /:\d*$/.test(url[1])) throw new Error(`_headers: an absolute rule needs an https host without a port and a path: ${line}`);+ rules.push({ host: url[1], path: url[2], set: {}, unset: [] });+ }else if (!rules.length) throw new Error(`_headers: a header before any path: ${line}`);else if (line.startsWith('! ')) rules.at(-1).unset.push(line.slice(2).trim().toLowerCase());else {@@ -353,11 +359,24 @@ const headerRuleRegExp = (path) => {return new RegExp(`^${pattern.replace(/:([A-Za-z]\w*)/g, '(?<$1>[^/]+)')}$`);};-/** The headers served for a path: matching rules in order; `!` removes, a name set twice is joined with a comma. */-export function headersFor(rules, pathname) {+/** A rule host as the host matches it: `:name` is one label, since a host placeholder ends at a period. */+const headerHostRegExp = (host) => {+ const labels = host.split('.').map((label) => (/^:[A-Za-z]\w*$/.test(label) ? '[^.]+' : label.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&')));+ return new RegExp(`^${labels.join('\\.')}$`, 'i');+};++/** The Pages project's own hosts, which must stay unindexed: its production alias and a deployment and a branch alias. */+export const PAGES_DEV_HOSTS = ['kilnstudio.pages.dev', '0123abcd.kilnstudio.pages.dev', 'production.kilnstudio.pages.dev'];++/**+ * The headers served for a path on `host` (the production host unless named): matching rules in order; `!` removes, a+ * name set twice is joined with a comma. Path rules apply on every host; an absolute rule only on the hosts it names.+ */+export function headersFor(rules, pathname, host = new URL(ORIGIN).host) {const headers = new Map();const seen = new Set();for (const rule of rules) {+ if (rule.host && !headerHostRegExp(rule.host).test(host)) continue;if (!headerRuleRegExp(rule.path).test(pathname)) continue;for (const name of rule.unset) headers.delete(name);for (const [name, value] of Object.entries(rule.set)) {@@ -398,6 +417,13 @@ export function headersErrors({ text, inlineScripts = [], files = [], sealedFileif (page['content-security-policy']) errors.push('The content security policy is enforced; it stays report-only until the owner decides');const policy = page['content-security-policy-report-only'];if (!policy) errors.push('Pages carry no Content-Security-Policy-Report-Only');+ // The Pages project also answers on its pages.dev hosts with the same bytes; only the production host is indexed.+ if (/noindex/i.test(page['x-robots-tag'] ?? '')) errors.push(`The production host ${new URL(ORIGIN).host} must not be served X-Robots-Tag: noindex`);+ for (const host of PAGES_DEV_HOSTS) {+ for (const path of ['/', '/docs/', '/robots.txt']) {+ if (!/noindex/i.test(headersFor(rules, path, host)['x-robots-tag'] ?? '')) errors.push(`https://${host}${path} must be served X-Robots-Tag: noindex`);+ }+ }const cache = page['cache-control'] ?? '';const maxAge = Number(cache.match(/max-age=(\d+)/)?.[1] ?? Number.NaN);if (!(maxAge <= 300) || /immutable/.test(cache)) errors.push(`HTML must be cached briefly and revalidated, got Cache-Control ${JSON.stringify(cache || null)}`);diff --git a/site/scripts/static-validation.test.ts b/site/scripts/static-validation.test.tsindex ab02e216..5fa835f1 100644--- a/site/scripts/static-validation.test.ts+++ b/site/scripts/static-validation.test.ts@@ -379,12 +379,18 @@ describe('the header layer in _headers (engineering review, finding 11)', () =>/scene-runtime/:scene/*.js! Cache-ControlCache-Control: public, max-age=31536000, immutable++https://kilnstudio.pages.dev/*+ X-Robots-Tag: noindex++https://:version.kilnstudio.pages.dev/*+ X-Robots-Tag: noindex${extra}`;const good = file(`default-src 'self'; script-src 'self' ${scriptHashSource(script)}`);test('rules apply in order, "!" removes what an earlier rule set, placeholders match one segment', () => {const rules = parseHeaderRules(good);- expect(rules.map((rule) => rule.path)).toEqual(['/*', '/_astro/*', '/scene-runtime/:scene/*.js']);+ expect(rules.map((rule) => `${rule.host ?? ''}${rule.path}`)).toEqual(['/*', '/_astro/*', '/scene-runtime/:scene/*.js', 'kilnstudio.pages.dev/*', ':version.kilnstudio.pages.dev/*']);expect(headersFor(rules, '/')['cache-control']).toBe('public, max-age=0, must-revalidate');expect(headersFor(rules, '/_astro/viewer.GoKKfp1n.js')['cache-control']).toBe('public, max-age=31536000, immutable');expect(headersFor(rules, '/scene-runtime/golden-gate/index-CA13LcNQ.js')['cache-control']).toBe('public, max-age=31536000, immutable');@@ -424,6 +430,26 @@ ${extra}`;expect(headersErrors({ text: good.replace('max-age=0, must-revalidate', 'max-age=86400'), inlineScripts: [script] })).toEqual(['HTML must be cached briefly and revalidated, got Cache-Control "public, max-age=86400"']);});+ test('absolute rules apply only on the hosts they name: the pages.dev hosts are noindex, the production host is not', () => {+ const rules = parseHeaderRules(good);+ expect(headersFor(rules, '/docs/')['x-robots-tag']).toBeUndefined();+ expect(headersFor(rules, '/docs/', 'kilnstudio.tools')['x-robots-tag']).toBeUndefined();+ expect(headersFor(rules, '/docs/', 'kilnstudio.pages.dev')['x-robots-tag']).toBe('noindex');+ expect(headersFor(rules, '/', 'a1b2c3d4.kilnstudio.pages.dev')['x-robots-tag']).toBe('noindex');+ expect(headersFor(rules, '/', 'pr-7.kilnstudio.pages.dev')['x-robots-tag']).toBe('noindex');+ expect(headersFor(rules, '/', 'a.b.kilnstudio.pages.dev')['x-robots-tag']).toBeUndefined();+ expect(headersFor(rules, '/', 'other.pages.dev')['x-robots-tag']).toBeUndefined();+ expect(headersFor(rules, '/docs/', 'kilnstudio.pages.dev')['x-content-type-options']).toBe('nosniff');+ expect(() => parseHeaderRules('https://kilnstudio.pages.dev:8443/*\n X-Robots-Tag: noindex')).toThrow('without a port');+ });++ test('a pages.dev host left indexable, or a noindex production host, is reported', () => {+ const open = good.replace('\nhttps://kilnstudio.pages.dev/*\n X-Robots-Tag: noindex\n', '\n');+ expect(headersErrors({ text: open, inlineScripts: [script] })).toEqual(['/', '/docs/', '/robots.txt'].map((path) => `https://kilnstudio.pages.dev${path} must be served X-Robots-Tag: noindex`));+ const everywhere = `${good}\n/*\n X-Robots-Tag: noindex\n`;+ expect(headersErrors({ text: everywhere, inlineScripts: [script] })).toEqual(['The production host kilnstudio.tools must not be served X-Robots-Tag: noindex']);+ });+test('inline scripts are found in pages; JSON-LD data blocks are not scripts to a CSP', () => {const page = inspectHtml(`<html><head><script>${script}</script><script type="application/ld+json">{"@type":"Thing"}</script><script type="module" src="/_astro/a.js"></script></head><body></body></html>`);expect(page.inlineScripts).toEqual([script]);diff --git a/site/scripts/validate-static.mjs b/site/scripts/validate-static.mjsindex efb5b7fb..5558c8a6 100644--- a/site/scripts/validate-static.mjs+++ b/site/scripts/validate-static.mjs@@ -9,6 +9,7 @@ import {sealedTroyCache} from './sealed-cache.mjs';import {viewerModelInput} from './viewer-model-input.mjs';import { glbJsonText } from './private-data.mjs';import {validatePublicText} from './validate-public-text.mjs';+import { sitemapLastmodErrors } from './sitemap-routes.mjs';const site = resolve(dirname(fileURLToPath(import.meta.url)), '..');const args = process.argv.slice(2);@@ -130,6 +131,7 @@ let sitemapUrls = [];try {const sitemap = await readFile(resolve(root, 'sitemap.xml'), 'utf8');sitemapUrls = Array.from(sitemap.matchAll(/<loc>(.*?)<\/loc>/g), (match) => match[1]);+ for (const error of sitemapLastmodErrors(sitemap)) add('/sitemap.xml', 'sitemap', error);for (const [route, page] of pages) {const url = new URL(route, ORIGIN).href;if (page.noindex && sitemapUrls.includes(url)) add(route, 'sitemap', 'noindex page is in the sitemap');