AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(seo): avoid false page dates in shallow clones (#1476)
bbansdc9bef24310d
diff --git a/.github/workflows/seo.yml b/.github/workflows/seo.ymlindex c4a017c50..0c8b1c932 100644--- a/.github/workflows/seo.yml+++ b/.github/workflows/seo.yml@@ -62,6 +62,10 @@ jobs:- name: Article frontmatter schema unit testsrun: python3 scripts/seo/test_article_schema.py+ # #1476: shallow clones must not date pages from the boundary commit.+ - name: Date-source unit tests (full, shallow, untracked, no repo)+ run: python3 scripts/seo/test_urlmap.py+- name: generated articles are currentrun: python3 scripts/seo/build_articles.py --checkdiff --git a/scripts/seo/gen_sitemap.py b/scripts/seo/gen_sitemap.pyindex 6299e4718..4d7bc353d 100755--- a/scripts/seo/gen_sitemap.py+++ b/scripts/seo/gen_sitemap.py@@ -108,7 +108,8 @@ def main(argv: list[str] | None = None) -> int:root = pathlib.Path(args.root) if args.root else urlmap.landing_dir()dates = urlmap.DateSource(urlmap.repo_root())if not dates.git_ok:- print("WARNING: git is unusable here — every <lastmod> below falls back "+ print("WARNING: git history is unavailable or shallow — "+ "every <lastmod> below falls back ""to the filesystem mtime, which Google will not be able to verify",file=sys.stderr)diff --git a/scripts/seo/test_urlmap.py b/scripts/seo/test_urlmap.pynew file mode 100644index 000000000..2abd2d620--- /dev/null+++ b/scripts/seo/test_urlmap.py@@ -0,0 +1,75 @@+#!/usr/bin/env python3+"""Date-source regressions. Run: python3 scripts/seo/test_urlmap.py."""++import datetime as dt+import os+import pathlib+import subprocess+import tempfile+import unittest+import xml.etree.ElementTree as ET++import gen_sitemap+import urlmap+++class DateSourceTests(unittest.TestCase):+ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.addCleanup(self.tmp.cleanup)+ self.root = pathlib.Path(self.tmp.name)+ self.repo = self.root / "source"+ self.git("init", str(self.repo))+ self.git("-C", str(self.repo), "config", "user.name", "Date test")+ self.git("-C", str(self.repo), "config", "user.email", "[email protected]")+ self.page = self.repo / "page.html"+ self.page.write_text("original page", encoding="utf-8")+ self.commit("page", "2020-01-02T12:00:00+00:00")+ (self.repo / "other.txt").write_text("unrelated", encoding="utf-8")+ self.commit("unrelated", "2021-03-04T12:00:00+00:00")++ @staticmethod+ def git(*args, env=None):+ return subprocess.run(["git", *args], check=True, capture_output=True,+ text=True, env=env).stdout.strip()++ def commit(self, message, date):+ self.git("-C", str(self.repo), "add", ".")+ env = {**os.environ, "GIT_AUTHOR_DATE": date, "GIT_COMMITTER_DATE": date}+ self.git("-C", str(self.repo), "commit", "-m", message, env=env)++ def test_full_history_uses_actual_page_commit(self):+ dates = urlmap.DateSource(self.repo)+ self.assertEqual(dates.last_modified(self.page), "2020-01-02")+ self.assertEqual(dates.first_published(self.page), "2020-01-02")+ self.assertEqual(dates.fallbacks, [])++ def test_shallow_history_records_mtime_fallback(self):+ shallow = self.root / "shallow"+ self.git("clone", "--depth", "1", self.repo.as_uri(), str(shallow))+ page = shallow / "page.html"+ stamp = dt.datetime(2022, 5, 6, 12, tzinfo=dt.timezone.utc).timestamp()+ os.utime(page, (stamp, stamp))+ dates = urlmap.DateSource(shallow)+ self.assertEqual(dates.last_modified(page), "2022-05-06")+ self.assertEqual(dates.first_published(page), "2022-05-06")+ self.assertIn(str(page), dates.fallbacks)+ sitemap = ET.fromstring(gen_sitemap.build(shallow, dates))+ self.assertEqual(sitemap.findtext(".//{*}lastmod"), "2022-05-06")++ def test_untracked_page_records_mtime_fallback(self):+ page = self.repo / "new.html"+ page.write_text("untracked", encoding="utf-8")+ dates = urlmap.DateSource(self.repo)+ self.assertEqual(dates.last_modified(page), dates._mtime_date(page))+ self.assertEqual(dates.fallbacks, [str(page)])++ def test_no_repository_records_mtime_fallback(self):+ dates = urlmap.DateSource(self.root)+ self.assertFalse(dates.git_ok)+ self.assertEqual(dates.last_modified(self.page), dates._mtime_date(self.page))+ self.assertEqual(dates.fallbacks, [str(self.page)])+++if __name__ == "__main__":+ unittest.main()diff --git a/scripts/seo/urlmap.py b/scripts/seo/urlmap.pyindex f609d0f2f..0675d6316 100755--- a/scripts/seo/urlmap.py+++ b/scripts/seo/urlmap.py@@ -292,10 +292,14 @@ class DateSource:def _probe(self) -> bool:try:r = subprocess.run(- ["git", "-C", str(self.repo_root), "rev-parse", "--git-dir"],+ ["git", "-C", str(self.repo_root), "rev-parse",+ "--is-shallow-repository"],capture_output=True, text=True, timeout=15,)- return r.returncode == 0+ # At a shallow boundary, git treats every surviving file as added+ # by that commit, even if it was last changed earlier. Those dates+ # are not verifiable page history; use the recorded mtime fallback.+ return r.returncode == 0 and r.stdout.strip() == "false"except (OSError, subprocess.SubprocessError):return False