AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
test(site): verify agent discovery after deployment (#2299)
Mario Marcolongo4526f2bf4103
diff --git a/.github/workflows/seo.yml b/.github/workflows/seo.ymlindex 866875882..0fed63c8e 100644--- a/.github/workflows/seo.yml+++ b/.github/workflows/seo.yml@@ -66,6 +66,10 @@ jobs:- name: Date-source unit tests (full, shallow, untracked, no repo)run: python3 scripts/seo/test_urlmap.py+ # Fixture HTTP server only: production verification runs after deploy.+ - name: Agent-discovery delivery regressions+ run: python3 scripts/seo/test_verify_agent_discovery.py+# The field report has its own section, ahead of Optional, and the# homepage prompt asks before sending.- name: Field-report section and permission linediff --git a/CHANGELOG.md b/CHANGELOG.mdindex 339e53386..410603866 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -718,6 +718,10 @@ samuelmbabhazi, elewashy, WYSTAF, kamwoh, mgtf, mariomarcolongo and Jun516.[#2137](https://github.com/xerj-org/xerj/pull/2137), Jarvis636431; theprefix half of #2080).+- Read-only post-deployment agent-discovery verifier for final GET/HEAD Link+ headers, documentation targets and optional public-skill digest checks.+ CI exercises local HTTP fixtures rather than contacting production.+### Fixed- `xerj autoindex` updates a brain's `nodes_index` when this run's datasetdiff --git a/scripts/seo/README.md b/scripts/seo/README.mdindex c5a75b7c2..0fe70343d 100644--- a/scripts/seo/README.md+++ b/scripts/seo/README.md@@ -614,6 +614,32 @@ touch `claims_rules.py` or `scripts/seo/testdata/`.### The check that must NOT be a gate+`verify_agent_discovery.py` checks the agent-discovery contract after deployment:++```sh+python3 scripts/seo/verify_agent_discovery.py+python3 scripts/seo/verify_agent_discovery.py https://staging.example.com --json+# Opt in only on a deployment expected to publish the public xerj skill:+python3 scripts/seo/verify_agent_discovery.py --skills --json+```++The default checks five requests: homepage GET and HEAD, then the three fixed+documentation targets from its discovery headers. It requires the relations+on the final response, not just in 103 Early Hints; a target must return 200+with the expected media type, and OpenAPI JSON must have a version and paths.+`--skills` additionally checks the v0.2.0 index, the public `xerj` entry, CORS,+and the exact-byte SHA-256 of its single-file artifact. It checks consistency,+not authenticity, and does not validate other skills or activate instructions.++Only fixed paths on the chosen origin are fetched. Redirects are reported,+never followed; responses are capped at 1 MiB. Requests are direct HTTP(S),+without proxy configuration, credentials or cookies. `--timeout` limits each+socket operation (not total run time). Exit 0 means pass, 1 means a delivery+or contract failure, and 2 means invalid CLI arguments. Wait for deployment+propagation before treating a failure as a code defect. The tool reads no+XERJ node or user corpus and changes no remote state. CI runs only its+loopback fixtures: `python3 scripts/seo/test_verify_agent_discovery.py`.+`verify_live.py` needs a deployment, so on a pull request every URL it asks for404s. Wiring it into `seo.yml` would make the merge gate fail on correct work,which is how gates get ignored. Run it **after** a deploy:diff --git a/scripts/seo/test_verify_agent_discovery.py b/scripts/seo/test_verify_agent_discovery.pynew file mode 100644index 000000000..4cad0e8aa--- /dev/null+++ b/scripts/seo/test_verify_agent_discovery.py@@ -0,0 +1,225 @@+#!/usr/bin/env python3+"""Exercise delivery failures over loopback HTTP; never contacts production."""++import contextlib+import hashlib+import http.server+import io+import json+import threading+import unittest+from email.message import Message+from unittest import mock++import verify_agent_discovery as verifier++LINKS = ', '.join(f'<{path}>; rel="{rel}"; type="{media}"'+ for rel, (path, media) in verifier.TARGETS.items())+SKILL = b'---\r\nname: xerj\r\ndescription: "Fixture"\r\n---\r\n# Skill\r\n'+++class Handler(http.server.BaseHTTPRequestHandler):+ def log_message(self, *args):+ pass++ def do_HEAD(self):+ self.do_GET()++ def do_GET(self):+ self.server.requests.append((self.command, self.path))+ status, headers, body = self.server.routes.get(self.path, (404, {}, b"missing"))+ if self.path == "/" and self.server.early:+ self.wfile.write(b'HTTP/1.1 103 Early Hints\r\nLink: ' ++ self.server.early.encode() + b'\r\n\r\n')+ self.send_response(status)+ for key, value in headers.items():+ for item in value if isinstance(value, list) else [value]:+ self.send_header(key, item)+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ if self.command != "HEAD":+ self.wfile.write(body)+++class DeliveryTests(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ cls.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ cls.server.daemon_threads = True+ cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)+ cls.thread.start()+ cls.base = "http://127.0.0.1:" + str(cls.server.server_port)++ @classmethod+ def tearDownClass(cls):+ cls.server.shutdown()+ cls.server.server_close()+ cls.thread.join()++ def setUp(self):+ self.server.early = None+ self.server.requests = []+ self.index = {"$schema": verifier.SKILL_SCHEMA, "skills": [{+ "name": "xerj", "type": "skill-md", "description": "Fixture",+ "url": verifier.SKILL_FILE,+ "digest": "sha256:" + hashlib.sha256(SKILL).hexdigest(),+ }]}+ self.server.routes = {+ "/": (200, {"Content-Type": "text/html", "Link": LINKS}, b"<html>home</html>"),+ "/llms.txt": (200, {"Content-Type": "text/plain; charset=utf-8"}, b"# XERJ"),+ "/openapi.json": (200, {"Content-Type": "application/json"},+ b'{"openapi":"3.1.0","paths":{}}'),+ "/docs/agents/endpoints": (200, {"Content-Type": "text/html"}, b"<h1>Endpoints</h1>"),+ verifier.SKILL_FILE: (200, {"Content-Type": "text/markdown; charset=utf-8",+ "Access-Control-Allow-Origin": "*"}, SKILL),+ }+ self.publish_index()++ def publish_index(self):+ self.server.routes[verifier.SKILL_INDEX] = (+ 200, {"Content-Type": "application/json", "Access-Control-Allow-Origin": "*"},+ json.dumps(self.index).encode())++ def failed(self, skills=False):+ return [r for r in verifier.verify(self.base, skills=skills, timeout=2) if not r["ok"]]++ def test_success_and_read_only_request_scope(self):+ self.assertEqual(self.failed(skills=True), [])+ self.assertEqual(self.server.requests, [("GET", "/"), ("HEAD", "/"),+ ("GET", "/llms.txt"), ("GET", "/openapi.json"),+ ("GET", "/docs/agents/endpoints"),+ ("GET", verifier.SKILL_INDEX), ("GET", verifier.SKILL_FILE)])++ def test_missing_links_fail_get_and_head(self):+ self.server.routes["/"][1].pop("Link")+ failures = self.failed()+ self.assertEqual([r["method"] for r in failures], ["GET", "HEAD"])+ self.assertIn("missing service-desc", ' '.join(failures[0]["failures"]))++ def test_early_hints_cannot_satisfy_final_response(self):+ self.server.early = LINKS+ self.server.routes["/"][1]["Link"] = '<https://fonts.example>; rel="preconnect"'+ self.assertEqual(len(self.failed()), 2)++ def test_early_hints_followed_by_valid_final_headers(self):+ self.server.early = '<https://fonts.example>; rel="preconnect"'+ self.server.routes["/"][1]["Link"] = LINKS.split(', ')+ self.assertEqual(self.failed(), [])++ def test_bad_target_delivery_is_a_failure(self):+ for status, media, body in [(404, "text/plain", b"missing"),+ (200, "text/html", b"<html>soft fallback</html>")]:+ with self.subTest(status=status, media=media):+ self.server.routes["/llms.txt"] = (status, {"Content-Type": media}, body)+ failures = self.failed()+ self.assertEqual(len(failures), 1)+ self.assertEqual(failures[0]["url"], self.base + "/llms.txt")++ def test_invalid_openapi_is_not_a_successful_json_response(self):+ for body in (b"not JSON", b"[]", b'{}'):+ with self.subTest(body=body):+ self.server.routes["/openapi.json"] = (200, {"Content-Type": "application/json"}, body)+ self.assertEqual(len(self.failed()), 1)++ def test_missing_content_type_is_not_inferred_as_plain_text(self):+ self.server.routes["/llms.txt"][1].pop("Content-Type")+ self.assertEqual(len(self.failed()), 1)++ def test_redirects_are_reported_without_following_them(self):+ self.server.routes["/llms.txt"] = (302, {"Location": "http://127.0.0.1:1/private"}, b"")+ self.assertEqual(len(self.failed()), 1)+ self.assertEqual(len(self.server.requests), 5)+ self.assertNotIn(("GET", "/private"), self.server.requests)++ def test_skill_check_is_explicitly_opt_in(self):+ self.server.routes.pop(verifier.SKILL_INDEX)+ self.assertEqual(self.failed(), [])+ self.assertEqual(len(self.failed(skills=True)), 1)++ def test_changed_skill_bytes_fail_the_advertised_digest(self):+ status, headers, body = self.server.routes[verifier.SKILL_FILE]+ self.server.routes[verifier.SKILL_FILE] = status, headers, body.replace(b"\r\n", b"\n")+ failures = self.failed(skills=True)+ self.assertEqual(len(failures), 1)+ self.assertIn("skill bytes do not match", failures[0]["failures"][0])++ def test_invalid_index_never_drives_arbitrary_requests(self):+ for key, value in [("url", "http://127.0.0.1:1/private"),+ ("url", "/get"), ("digest", "sha256:bad"),+ ("description", ""), ("type", "archive")]:+ with self.subTest(key=key, value=value):+ self.setUp()+ self.index["skills"][0][key] = value+ self.publish_index()+ self.assertEqual(len(self.failed(skills=True)), 1)+ self.assertEqual(len(self.server.requests), 6)++ def test_duplicate_or_malformed_index(self):+ for index in ([], {"skills": []}, {"$schema": verifier.SKILL_SCHEMA, "skills": []},+ {"$schema": verifier.SKILL_SCHEMA, "skills": self.index["skills"] * 2}):+ with self.subTest(index=index):+ self.index = index+ self.publish_index()+ self.assertEqual(len(self.failed(skills=True)), 1)++ def test_cors_and_media_types_are_checked(self):+ self.server.routes[verifier.SKILL_FILE][1].pop("Access-Control-Allow-Origin")+ self.assertEqual(len(self.failed(skills=True)), 1)+ self.publish_index()+ self.server.routes[verifier.SKILL_INDEX][1]["Content-Type"] = "text/html"+ self.assertEqual(len(self.failed(skills=True)), 1)++ def test_bounded_response_read(self):+ self.server.routes["/llms.txt"] = (200, {}, b"x" * 40)+ with mock.patch.object(verifier, "MAX_BYTES", 20):+ status, _, _, error = verifier.fetch(self.base, "/llms.txt", "GET", 2)+ self.assertEqual(status, 0)+ self.assertIn("verification limit", error)++ def test_json_cli_and_exit_codes(self):+ for broken, expected in [(False, 0), (True, 1)]:+ if broken:+ self.server.routes["/"][1].pop("Link")+ output = io.StringIO()+ with contextlib.redirect_stdout(output):+ code = verifier.main([self.base, "--json"])+ report = json.loads(output.getvalue())+ self.assertEqual(code, expected)+ self.assertEqual(report["failed"], 2 if broken else 0)+ self.assertEqual(report["checked"], 5)+++class ParserTests(unittest.TestCase):+ def test_quoted_commas_multiple_relations_and_duplicate_parameters(self):+ headers = Message()+ headers["Link"] = ('</a,b>; title="quoted, \\"title\\""; rel="service-doc describedby", '+ '</x>; rel="alternate"; rel="service-desc"')+ found = verifier.relations(headers, "https://example.test")+ self.assertIn(("service-doc", "https://example.test/a,b"), found)+ self.assertIn(("describedby", "https://example.test/a,b"), found)+ self.assertNotIn(("service-desc", "https://example.test/x"), found)++ def test_anchored_or_quoted_text_does_not_forge_homepage_links(self):+ headers = Message()+ headers["Link"] = '</x>; rel="service-desc"; anchor="/elsewhere"'+ self.assertEqual(verifier.relations(headers, "https://example.test"), set())+ headers.replace_header("Link", '</x>; title="<fake>; rel=service-desc"')+ self.assertEqual(verifier.relations(headers, "https://example.test"), set())+ headers.replace_header("Link", '</x>; rel="unclosed')+ with self.assertRaises(ValueError):+ verifier.relations(headers, "https://example.test")++ def test_invalid_origins_stop_before_any_request(self):+ for value in ("file:///tmp/data", "https://u:[email protected]", "https://example.test/path",+ "https://example.test?q=x", "https://example.test#fragment",+ "https://example.test:99999", "https://exam ple.test"):+ with self.subTest(value=value), mock.patch.object(verifier, "fetch") as fetch, \+ contextlib.redirect_stderr(io.StringIO()):+ with self.assertRaises(SystemExit) as error:+ verifier.main([value])+ self.assertEqual(error.exception.code, 2)+ fetch.assert_not_called()+++if __name__ == "__main__":+ unittest.main()diff --git a/scripts/seo/verify_agent_discovery.py b/scripts/seo/verify_agent_discovery.pynew file mode 100644index 000000000..5a316a1e8--- /dev/null+++ b/scripts/seo/verify_agent_discovery.py@@ -0,0 +1,227 @@+#!/usr/bin/env python3+"""Read-only post-deploy verification of XERJ's agent-discovery resources.++Run [origin] [--skills] [--json]. Exit 0: pass; 1: delivery/contract failure;+2: usage error. Live checks belong after deployment, never in a PR merge gate.+Only fixed XERJ resource paths are requested; advertised URLs are never fetched.+Requests are direct HTTP(S), without cookies, credentials, proxies or redirects.+"""++from __future__ import annotations++import argparse+import hashlib+import http.client+import json+import re+import sys+import urllib.parse+from email.message import Message++from verify_live import USER_AGENT++MAX_BYTES = 1024 * 1024+TARGETS = {+ "describedby": ("/llms.txt", "text/plain"),+ "service-desc": ("/openapi.json", "application/json"),+ "service-doc": ("/docs/agents/endpoints", "text/html"),+}+SKILL_INDEX = "/.well-known/agent-skills/index.json"+SKILL_FILE = "/.well-known/agent-skills/xerj/SKILL.md"+SKILL_SCHEMA = "https://schemas.agentskills.io/discovery/0.2.0/schema.json"+++class FinalResponse(http.client.HTTPResponse):+ def _read_status(self):+ # HTTPResponse in supported Python versions skips 100, but can expose+ # 103 as the final response. Consume informational headers separately;+ # only the final response's Link fields can satisfy discovery checks.+ # RFC 9110 section 15.2; bounded against endless informational replies.+ for _ in range(9):+ version, status, reason = super()._read_status()+ if not 100 <= status < 200 or status == 101:+ return version, status, reason+ http.client.parse_headers(self.fp)+ raise http.client.HTTPException("too many informational responses")+++def origin(value: str) -> str:+ p = urllib.parse.urlsplit(value)+ if (p.scheme not in {"http", "https"} or not p.hostname+ or p.username is not None or p.password is not None+ or p.path not in {"", "/"} or p.query or p.fragment+ or any(c.isspace() for c in value)):+ raise ValueError("expected an http(s) origin without credentials, path, query or fragment")+ _ = p.port # validate malformed/out-of-range ports before any request+ return urllib.parse.urlunsplit((p.scheme, p.netloc, "", "", ""))+++def fetch(base: str, path: str, method: str, timeout: float):+ p = urllib.parse.urlsplit(base)+ cls = http.client.HTTPSConnection if p.scheme == "https" else http.client.HTTPConnection+ conn = cls(p.hostname, p.port, timeout=timeout)+ conn.response_class = FinalResponse+ try:+ conn.request(method, path, headers={"User-Agent": USER_AGENT,+ "Accept-Encoding": "identity"})+ response = conn.getresponse()+ body = response.read(MAX_BYTES + 1)+ if len(body) > MAX_BYTES:+ raise ValueError("response exceeds the 1 MiB verification limit")+ return response.status, response.headers, body, None+ except (OSError, ValueError, http.client.HTTPException) as exc:+ return 0, Message(), b"", f"{type(exc).__name__}: {exc}"+ finally:+ conn.close()+++def split_links(value: str) -> list[str]:+ """Split commas outside URI brackets and quoted strings (RFC 8288)."""+ parts, start, quoted, escaped, bracket = [], 0, False, False, False+ for i, char in enumerate(value):+ if escaped:+ escaped = False+ elif quoted and char == "\\":+ escaped = True+ elif char == '"' and not bracket:+ quoted = not quoted+ elif not quoted:+ if char == "<":+ bracket = True+ elif char == ">":+ bracket = False+ elif char == "," and not bracket:+ parts.append(value[start:i])+ start = i + 1+ if quoted or bracket:+ raise ValueError("unclosed Link URI or quoted string")+ return parts + [value[start:]]+++def relations(headers: Message, base: str) -> set[tuple[str, str]]:+ found = set()+ for value in headers.get_all("Link", []):+ for link in split_links(value):+ match = re.fullmatch(r"\s*<([^<>]*)>\s*(;.*)?\s*", link)+ if not match:+ raise ValueError("malformed Link field")+ params = Message()+ params["Content-Type"] = "link" + (match[2] or "")+ # get_param takes the first rel; a later duplicate must not turn+ # an unrelated link into a valid discovery relation (RFC 8288).+ rel = params.get_param("rel", "")+ if not isinstance(rel, str) or params.get_param("anchor") is not None:+ continue # an explicit anchor changes the link's context+ target = urllib.parse.urljoin(base + "/", match[1])+ for name in rel.split():+ found.add((name.lower(), target))+ return found+++def verify(base: str, *, skills: bool = False, timeout: float = 15) -> list[dict]:+ results = []++ def request(path, method="GET", media=None):+ status, headers, body, error = fetch(base, path, method, timeout)+ failures = [error] if error else []+ if not error and status != 200:+ failures.append(f"HTTP {status}; expected 200 (redirects are not followed)")+ if status == 200 and media and (len(headers.get_all("Content-Type", [])) != 1+ or headers.get_content_type() != media):+ failures.append(f"Content-Type {headers.get('Content-Type')!r}; expected {media}")+ row = {"method": method, "url": base + path, "status": status,+ "failures": failures}+ results.append(row)+ return row, headers, body++ for method in ("GET", "HEAD"):+ row, headers, _ = request("/", method, "text/html")+ if row["status"] == 200:+ try:+ found = relations(headers, base)+ for rel, (path, _) in TARGETS.items():+ if (rel, base + path) not in found:+ row["failures"].append(f"missing {rel} link to {path} on final response")+ except ValueError as exc:+ row["failures"].append(str(exc))++ for rel, (path, media) in TARGETS.items():+ row, _, body = request(path, media=media)+ if rel == "service-desc" and not row["failures"]:+ try:+ spec = json.loads(body)+ if not isinstance(spec, dict) or not isinstance(spec.get("openapi"), str) \+ or not isinstance(spec.get("paths"), dict):+ raise ValueError("missing OpenAPI version or paths object")+ except (ValueError, UnicodeError) as exc:+ row["failures"].append(f"invalid OpenAPI JSON: {exc}")++ if skills:+ row, headers, body = request(SKILL_INDEX, media="application/json")+ for key, expected in (("Access-Control-Allow-Origin", "*"),):+ if row["status"] == 200 and headers.get(key) != expected:+ row["failures"].append(f"{key} must be {expected}")+ digest = None+ if not row["failures"]:+ try:+ index = json.loads(body)+ if not isinstance(index, dict) or index.get("$schema") != SKILL_SCHEMA \+ or not isinstance(index.get("skills"), list):+ raise ValueError("expected the discovery v0.2.0 schema and skills array")+ entries = [e for e in index["skills"] if isinstance(e, dict) and e.get("name") == "xerj"]+ if len(entries) != 1:+ raise ValueError("expected exactly one xerj entry")+ entry = entries[0]+ if (entry.get("type") != "skill-md"+ or not isinstance(entry.get("url"), str)+ or urllib.parse.urljoin(base + SKILL_INDEX, entry["url"]) != base + SKILL_FILE+ or not isinstance(entry.get("description"), str)+ or not 1 <= len(entry["description"].strip()) <= 1024+ or not isinstance(entry.get("digest"), str)+ or not re.fullmatch(r"sha256:[0-9a-f]{64}", entry["digest"])):+ raise ValueError("invalid xerj skill metadata, URL or digest")+ digest = entry["digest"]+ except (ValueError, UnicodeError) as exc:+ row["failures"].append(f"invalid skill index: {exc}")+ if digest is not None:+ row, headers, body = request(SKILL_FILE, media="text/markdown")+ if row["status"] == 200:+ if headers.get("Access-Control-Allow-Origin") != "*":+ row["failures"].append("Access-Control-Allow-Origin must be *")+ if "sha256:" + hashlib.sha256(body).hexdigest() != digest:+ row["failures"].append("skill bytes do not match the index SHA-256")++ for row in results:+ row["ok"] = not row["failures"]+ return results+++def main(argv=None) -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("base", nargs="?", default="https://xerj.org")+ parser.add_argument("--skills", action="store_true", help="also require the public xerj skill index and digest")+ parser.add_argument("--timeout", type=float, default=15, help="per socket operation, seconds (0 < n <= 60)")+ parser.add_argument("--json", action="store_true")+ args = parser.parse_args(argv)+ try:+ base = origin(args.base)+ if not 0 < args.timeout <= 60:+ raise ValueError("timeout must be greater than 0 and at most 60 seconds")+ except ValueError as exc:+ parser.error(str(exc))+ results = verify(base, skills=args.skills, timeout=args.timeout)+ failed = sum(not row["ok"] for row in results)+ report = {"base": base, "skills": args.skills, "checked": len(results),+ "failed": failed, "results": results}+ if args.json:+ print(json.dumps(report, indent=2))+ else:+ for row in results:+ print(f"{'PASS' if row['ok'] else 'FAIL'} {row['method']} {row['url']} "+ f"HTTP {row['status']} " + json.dumps(row["failures"], ensure_ascii=True))+ print(f"verify_agent_discovery: {len(results)} requests, {failed} failed")+ return 1 if failed else 0+++if __name__ == "__main__":+ sys.exit(main())