Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(seo): avoid false page dates in shallow clones (#1476)

bbansdc9bef24310d
diff --git a/.github/workflows/seo.yml b/.github/workflows/seo.yml
index c4a017c50..0c8b1c932 100644
--- a/.github/workflows/seo.yml
+++ b/.github/workflows/seo.yml
@@ -62,6 +62,10 @@ jobs:
- name: Article frontmatter schema unit tests
run: python3 scripts/seo/test_article_schema.py
+ # #1476: shallow clones must not date pages from the boundary commit.
+ - name: Date-source unit tests (full, shallow, untracked, no repo)
+ run: python3 scripts/seo/test_urlmap.py
+
- name: generated articles are current
run: python3 scripts/seo/build_articles.py --check
diff --git a/scripts/seo/gen_sitemap.py b/scripts/seo/gen_sitemap.py
index 6299e4718..4d7bc353d 100755
--- a/scripts/seo/gen_sitemap.py
+++ b/scripts/seo/gen_sitemap.py
@@ -108,7 +108,8 @@ def main(argv: list[str] | None = None) -> int:
root = pathlib.Path(args.root) if args.root else urlmap.landing_dir()
dates = urlmap.DateSource(urlmap.repo_root())
if not dates.git_ok:
- print("WARNING: git is unusable here — every <lastmod> below falls back "
+ print("WARNING: git history is unavailable or shallow — "
+ "every <lastmod> below falls back "
"to the filesystem mtime, which Google will not be able to verify",
file=sys.stderr)
diff --git a/scripts/seo/test_urlmap.py b/scripts/seo/test_urlmap.py
new file mode 100644
index 000000000..2abd2d620
--- /dev/null
+++ b/scripts/seo/test_urlmap.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""Date-source regressions. Run: python3 scripts/seo/test_urlmap.py."""
+
+import datetime as dt
+import os
+import pathlib
+import subprocess
+import tempfile
+import unittest
+import xml.etree.ElementTree as ET
+
+import gen_sitemap
+import urlmap
+
+
+class DateSourceTests(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+ self.root = pathlib.Path(self.tmp.name)
+ self.repo = self.root / "source"
+ self.git("init", str(self.repo))
+ self.git("-C", str(self.repo), "config", "user.name", "Date test")
+ self.git("-C", str(self.repo), "config", "user.email", "[email protected]")
+ self.page = self.repo / "page.html"
+ self.page.write_text("original page", encoding="utf-8")
+ self.commit("page", "2020-01-02T12:00:00+00:00")
+ (self.repo / "other.txt").write_text("unrelated", encoding="utf-8")
+ self.commit("unrelated", "2021-03-04T12:00:00+00:00")
+
+ @staticmethod
+ def git(*args, env=None):
+ return subprocess.run(["git", *args], check=True, capture_output=True,
+ text=True, env=env).stdout.strip()
+
+ def commit(self, message, date):
+ self.git("-C", str(self.repo), "add", ".")
+ env = {**os.environ, "GIT_AUTHOR_DATE": date, "GIT_COMMITTER_DATE": date}
+ self.git("-C", str(self.repo), "commit", "-m", message, env=env)
+
+ def test_full_history_uses_actual_page_commit(self):
+ dates = urlmap.DateSource(self.repo)
+ self.assertEqual(dates.last_modified(self.page), "2020-01-02")
+ self.assertEqual(dates.first_published(self.page), "2020-01-02")
+ self.assertEqual(dates.fallbacks, [])
+
+ def test_shallow_history_records_mtime_fallback(self):
+ shallow = self.root / "shallow"
+ self.git("clone", "--depth", "1", self.repo.as_uri(), str(shallow))
+ page = shallow / "page.html"
+ stamp = dt.datetime(2022, 5, 6, 12, tzinfo=dt.timezone.utc).timestamp()
+ os.utime(page, (stamp, stamp))
+ dates = urlmap.DateSource(shallow)
+ self.assertEqual(dates.last_modified(page), "2022-05-06")
+ self.assertEqual(dates.first_published(page), "2022-05-06")
+ self.assertIn(str(page), dates.fallbacks)
+ sitemap = ET.fromstring(gen_sitemap.build(shallow, dates))
+ self.assertEqual(sitemap.findtext(".//{*}lastmod"), "2022-05-06")
+
+ def test_untracked_page_records_mtime_fallback(self):
+ page = self.repo / "new.html"
+ page.write_text("untracked", encoding="utf-8")
+ dates = urlmap.DateSource(self.repo)
+ self.assertEqual(dates.last_modified(page), dates._mtime_date(page))
+ self.assertEqual(dates.fallbacks, [str(page)])
+
+ def test_no_repository_records_mtime_fallback(self):
+ dates = urlmap.DateSource(self.root)
+ self.assertFalse(dates.git_ok)
+ self.assertEqual(dates.last_modified(self.page), dates._mtime_date(self.page))
+ self.assertEqual(dates.fallbacks, [str(self.page)])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/seo/urlmap.py b/scripts/seo/urlmap.py
index f609d0f2f..0675d6316 100755
--- a/scripts/seo/urlmap.py
+++ b/scripts/seo/urlmap.py
@@ -292,10 +292,14 @@ class DateSource:
def _probe(self) -> bool:
try:
r = subprocess.run(
- ["git", "-C", str(self.repo_root), "rev-parse", "--git-dir"],
+ ["git", "-C", str(self.repo_root), "rev-parse",
+ "--is-shallow-repository"],
capture_output=True, text=True, timeout=15,
)
- return r.returncode == 0
+ # At a shallow boundary, git treats every surviving file as added
+ # by that commit, even if it was last changed earlier. Those dates
+ # are not verifiable page history; use the recorded mtime fallback.
+ return r.returncode == 0 and r.stdout.strip() == "false"
except (OSError, subprocess.SubprocessError):
return False