Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

test(site): verify agent discovery after deployment (#2299)

Mario Marcolongo4526f2bf4103
diff --git a/.github/workflows/seo.yml b/.github/workflows/seo.yml
index 866875882..0fed63c8e 100644
--- a/.github/workflows/seo.yml
+++ b/.github/workflows/seo.yml
@@ -66,6 +66,10 @@ jobs:
- name: Date-source unit tests (full, shallow, untracked, no repo)
run: python3 scripts/seo/test_urlmap.py
+ # Fixture HTTP server only: production verification runs after deploy.
+ - name: Agent-discovery delivery regressions
+ run: python3 scripts/seo/test_verify_agent_discovery.py
+
# The field report has its own section, ahead of Optional, and the
# homepage prompt asks before sending.
- name: Field-report section and permission line
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 339e53386..410603866 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -718,6 +718,10 @@ samuelmbabhazi, elewashy, WYSTAF, kamwoh, mgtf, mariomarcolongo and Jun516.
[#2137](https://github.com/xerj-org/xerj/pull/2137), Jarvis636431; the
prefix half of #2080).
+- Read-only post-deployment agent-discovery verifier for final GET/HEAD Link
+ headers, documentation targets and optional public-skill digest checks.
+ CI exercises local HTTP fixtures rather than contacting production.
+
### Fixed
- `xerj autoindex` updates a brain's `nodes_index` when this run's dataset
diff --git a/scripts/seo/README.md b/scripts/seo/README.md
index c5a75b7c2..0fe70343d 100644
--- a/scripts/seo/README.md
+++ b/scripts/seo/README.md
@@ -614,6 +614,32 @@ touch `claims_rules.py` or `scripts/seo/testdata/`.
### The check that must NOT be a gate
+`verify_agent_discovery.py` checks the agent-discovery contract after deployment:
+
+```sh
+python3 scripts/seo/verify_agent_discovery.py
+python3 scripts/seo/verify_agent_discovery.py https://staging.example.com --json
+# Opt in only on a deployment expected to publish the public xerj skill:
+python3 scripts/seo/verify_agent_discovery.py --skills --json
+```
+
+The default checks five requests: homepage GET and HEAD, then the three fixed
+documentation targets from its discovery headers. It requires the relations
+on the final response, not just in 103 Early Hints; a target must return 200
+with the expected media type, and OpenAPI JSON must have a version and paths.
+`--skills` additionally checks the v0.2.0 index, the public `xerj` entry, CORS,
+and the exact-byte SHA-256 of its single-file artifact. It checks consistency,
+not authenticity, and does not validate other skills or activate instructions.
+
+Only fixed paths on the chosen origin are fetched. Redirects are reported,
+never followed; responses are capped at 1 MiB. Requests are direct HTTP(S),
+without proxy configuration, credentials or cookies. `--timeout` limits each
+socket operation (not total run time). Exit 0 means pass, 1 means a delivery
+or contract failure, and 2 means invalid CLI arguments. Wait for deployment
+propagation before treating a failure as a code defect. The tool reads no
+XERJ node or user corpus and changes no remote state. CI runs only its
+loopback fixtures: `python3 scripts/seo/test_verify_agent_discovery.py`.
+
`verify_live.py` needs a deployment, so on a pull request every URL it asks for
404s. Wiring it into `seo.yml` would make the merge gate fail on correct work,
which is how gates get ignored. Run it **after** a deploy:
diff --git a/scripts/seo/test_verify_agent_discovery.py b/scripts/seo/test_verify_agent_discovery.py
new file mode 100644
index 000000000..4cad0e8aa
--- /dev/null
+++ b/scripts/seo/test_verify_agent_discovery.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Exercise delivery failures over loopback HTTP; never contacts production."""
+
+import contextlib
+import hashlib
+import http.server
+import io
+import json
+import threading
+import unittest
+from email.message import Message
+from unittest import mock
+
+import verify_agent_discovery as verifier
+
+LINKS = ', '.join(f'<{path}>; rel="{rel}"; type="{media}"'
+ for rel, (path, media) in verifier.TARGETS.items())
+SKILL = b'---\r\nname: xerj\r\ndescription: "Fixture"\r\n---\r\n# Skill\r\n'
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+ def log_message(self, *args):
+ pass
+
+ def do_HEAD(self):
+ self.do_GET()
+
+ def do_GET(self):
+ self.server.requests.append((self.command, self.path))
+ status, headers, body = self.server.routes.get(self.path, (404, {}, b"missing"))
+ if self.path == "/" and self.server.early:
+ self.wfile.write(b'HTTP/1.1 103 Early Hints\r\nLink: ' +
+ self.server.early.encode() + b'\r\n\r\n')
+ self.send_response(status)
+ for key, value in headers.items():
+ for item in value if isinstance(value, list) else [value]:
+ self.send_header(key, item)
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ if self.command != "HEAD":
+ self.wfile.write(body)
+
+
+class DeliveryTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ cls.server.daemon_threads = True
+ cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
+ cls.thread.start()
+ cls.base = "http://127.0.0.1:" + str(cls.server.server_port)
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.server.shutdown()
+ cls.server.server_close()
+ cls.thread.join()
+
+ def setUp(self):
+ self.server.early = None
+ self.server.requests = []
+ self.index = {"$schema": verifier.SKILL_SCHEMA, "skills": [{
+ "name": "xerj", "type": "skill-md", "description": "Fixture",
+ "url": verifier.SKILL_FILE,
+ "digest": "sha256:" + hashlib.sha256(SKILL).hexdigest(),
+ }]}
+ self.server.routes = {
+ "/": (200, {"Content-Type": "text/html", "Link": LINKS}, b"<html>home</html>"),
+ "/llms.txt": (200, {"Content-Type": "text/plain; charset=utf-8"}, b"# XERJ"),
+ "/openapi.json": (200, {"Content-Type": "application/json"},
+ b'{"openapi":"3.1.0","paths":{}}'),
+ "/docs/agents/endpoints": (200, {"Content-Type": "text/html"}, b"<h1>Endpoints</h1>"),
+ verifier.SKILL_FILE: (200, {"Content-Type": "text/markdown; charset=utf-8",
+ "Access-Control-Allow-Origin": "*"}, SKILL),
+ }
+ self.publish_index()
+
+ def publish_index(self):
+ self.server.routes[verifier.SKILL_INDEX] = (
+ 200, {"Content-Type": "application/json", "Access-Control-Allow-Origin": "*"},
+ json.dumps(self.index).encode())
+
+ def failed(self, skills=False):
+ return [r for r in verifier.verify(self.base, skills=skills, timeout=2) if not r["ok"]]
+
+ def test_success_and_read_only_request_scope(self):
+ self.assertEqual(self.failed(skills=True), [])
+ self.assertEqual(self.server.requests, [("GET", "/"), ("HEAD", "/"),
+ ("GET", "/llms.txt"), ("GET", "/openapi.json"),
+ ("GET", "/docs/agents/endpoints"),
+ ("GET", verifier.SKILL_INDEX), ("GET", verifier.SKILL_FILE)])
+
+ def test_missing_links_fail_get_and_head(self):
+ self.server.routes["/"][1].pop("Link")
+ failures = self.failed()
+ self.assertEqual([r["method"] for r in failures], ["GET", "HEAD"])
+ self.assertIn("missing service-desc", ' '.join(failures[0]["failures"]))
+
+ def test_early_hints_cannot_satisfy_final_response(self):
+ self.server.early = LINKS
+ self.server.routes["/"][1]["Link"] = '<https://fonts.example>; rel="preconnect"'
+ self.assertEqual(len(self.failed()), 2)
+
+ def test_early_hints_followed_by_valid_final_headers(self):
+ self.server.early = '<https://fonts.example>; rel="preconnect"'
+ self.server.routes["/"][1]["Link"] = LINKS.split(', ')
+ self.assertEqual(self.failed(), [])
+
+ def test_bad_target_delivery_is_a_failure(self):
+ for status, media, body in [(404, "text/plain", b"missing"),
+ (200, "text/html", b"<html>soft fallback</html>")]:
+ with self.subTest(status=status, media=media):
+ self.server.routes["/llms.txt"] = (status, {"Content-Type": media}, body)
+ failures = self.failed()
+ self.assertEqual(len(failures), 1)
+ self.assertEqual(failures[0]["url"], self.base + "/llms.txt")
+
+ def test_invalid_openapi_is_not_a_successful_json_response(self):
+ for body in (b"not JSON", b"[]", b'{}'):
+ with self.subTest(body=body):
+ self.server.routes["/openapi.json"] = (200, {"Content-Type": "application/json"}, body)
+ self.assertEqual(len(self.failed()), 1)
+
+ def test_missing_content_type_is_not_inferred_as_plain_text(self):
+ self.server.routes["/llms.txt"][1].pop("Content-Type")
+ self.assertEqual(len(self.failed()), 1)
+
+ def test_redirects_are_reported_without_following_them(self):
+ self.server.routes["/llms.txt"] = (302, {"Location": "http://127.0.0.1:1/private"}, b"")
+ self.assertEqual(len(self.failed()), 1)
+ self.assertEqual(len(self.server.requests), 5)
+ self.assertNotIn(("GET", "/private"), self.server.requests)
+
+ def test_skill_check_is_explicitly_opt_in(self):
+ self.server.routes.pop(verifier.SKILL_INDEX)
+ self.assertEqual(self.failed(), [])
+ self.assertEqual(len(self.failed(skills=True)), 1)
+
+ def test_changed_skill_bytes_fail_the_advertised_digest(self):
+ status, headers, body = self.server.routes[verifier.SKILL_FILE]
+ self.server.routes[verifier.SKILL_FILE] = status, headers, body.replace(b"\r\n", b"\n")
+ failures = self.failed(skills=True)
+ self.assertEqual(len(failures), 1)
+ self.assertIn("skill bytes do not match", failures[0]["failures"][0])
+
+ def test_invalid_index_never_drives_arbitrary_requests(self):
+ for key, value in [("url", "http://127.0.0.1:1/private"),
+ ("url", "/get"), ("digest", "sha256:bad"),
+ ("description", ""), ("type", "archive")]:
+ with self.subTest(key=key, value=value):
+ self.setUp()
+ self.index["skills"][0][key] = value
+ self.publish_index()
+ self.assertEqual(len(self.failed(skills=True)), 1)
+ self.assertEqual(len(self.server.requests), 6)
+
+ def test_duplicate_or_malformed_index(self):
+ for index in ([], {"skills": []}, {"$schema": verifier.SKILL_SCHEMA, "skills": []},
+ {"$schema": verifier.SKILL_SCHEMA, "skills": self.index["skills"] * 2}):
+ with self.subTest(index=index):
+ self.index = index
+ self.publish_index()
+ self.assertEqual(len(self.failed(skills=True)), 1)
+
+ def test_cors_and_media_types_are_checked(self):
+ self.server.routes[verifier.SKILL_FILE][1].pop("Access-Control-Allow-Origin")
+ self.assertEqual(len(self.failed(skills=True)), 1)
+ self.publish_index()
+ self.server.routes[verifier.SKILL_INDEX][1]["Content-Type"] = "text/html"
+ self.assertEqual(len(self.failed(skills=True)), 1)
+
+ def test_bounded_response_read(self):
+ self.server.routes["/llms.txt"] = (200, {}, b"x" * 40)
+ with mock.patch.object(verifier, "MAX_BYTES", 20):
+ status, _, _, error = verifier.fetch(self.base, "/llms.txt", "GET", 2)
+ self.assertEqual(status, 0)
+ self.assertIn("verification limit", error)
+
+ def test_json_cli_and_exit_codes(self):
+ for broken, expected in [(False, 0), (True, 1)]:
+ if broken:
+ self.server.routes["/"][1].pop("Link")
+ output = io.StringIO()
+ with contextlib.redirect_stdout(output):
+ code = verifier.main([self.base, "--json"])
+ report = json.loads(output.getvalue())
+ self.assertEqual(code, expected)
+ self.assertEqual(report["failed"], 2 if broken else 0)
+ self.assertEqual(report["checked"], 5)
+
+
+class ParserTests(unittest.TestCase):
+ def test_quoted_commas_multiple_relations_and_duplicate_parameters(self):
+ headers = Message()
+ headers["Link"] = ('</a,b>; title="quoted, \\"title\\""; rel="service-doc describedby", '
+ '</x>; rel="alternate"; rel="service-desc"')
+ found = verifier.relations(headers, "https://example.test")
+ self.assertIn(("service-doc", "https://example.test/a,b"), found)
+ self.assertIn(("describedby", "https://example.test/a,b"), found)
+ self.assertNotIn(("service-desc", "https://example.test/x"), found)
+
+ def test_anchored_or_quoted_text_does_not_forge_homepage_links(self):
+ headers = Message()
+ headers["Link"] = '</x>; rel="service-desc"; anchor="/elsewhere"'
+ self.assertEqual(verifier.relations(headers, "https://example.test"), set())
+ headers.replace_header("Link", '</x>; title="<fake>; rel=service-desc"')
+ self.assertEqual(verifier.relations(headers, "https://example.test"), set())
+ headers.replace_header("Link", '</x>; rel="unclosed')
+ with self.assertRaises(ValueError):
+ verifier.relations(headers, "https://example.test")
+
+ def test_invalid_origins_stop_before_any_request(self):
+ for value in ("file:///tmp/data", "https://u:[email protected]", "https://example.test/path",
+ "https://example.test?q=x", "https://example.test#fragment",
+ "https://example.test:99999", "https://exam ple.test"):
+ with self.subTest(value=value), mock.patch.object(verifier, "fetch") as fetch, \
+ contextlib.redirect_stderr(io.StringIO()):
+ with self.assertRaises(SystemExit) as error:
+ verifier.main([value])
+ self.assertEqual(error.exception.code, 2)
+ fetch.assert_not_called()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/seo/verify_agent_discovery.py b/scripts/seo/verify_agent_discovery.py
new file mode 100644
index 000000000..5a316a1e8
--- /dev/null
+++ b/scripts/seo/verify_agent_discovery.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""Read-only post-deploy verification of XERJ's agent-discovery resources.
+
+Run [origin] [--skills] [--json]. Exit 0: pass; 1: delivery/contract failure;
+2: usage error. Live checks belong after deployment, never in a PR merge gate.
+Only fixed XERJ resource paths are requested; advertised URLs are never fetched.
+Requests are direct HTTP(S), without cookies, credentials, proxies or redirects.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import http.client
+import json
+import re
+import sys
+import urllib.parse
+from email.message import Message
+
+from verify_live import USER_AGENT
+
+MAX_BYTES = 1024 * 1024
+TARGETS = {
+ "describedby": ("/llms.txt", "text/plain"),
+ "service-desc": ("/openapi.json", "application/json"),
+ "service-doc": ("/docs/agents/endpoints", "text/html"),
+}
+SKILL_INDEX = "/.well-known/agent-skills/index.json"
+SKILL_FILE = "/.well-known/agent-skills/xerj/SKILL.md"
+SKILL_SCHEMA = "https://schemas.agentskills.io/discovery/0.2.0/schema.json"
+
+
+class FinalResponse(http.client.HTTPResponse):
+ def _read_status(self):
+ # HTTPResponse in supported Python versions skips 100, but can expose
+ # 103 as the final response. Consume informational headers separately;
+ # only the final response's Link fields can satisfy discovery checks.
+ # RFC 9110 section 15.2; bounded against endless informational replies.
+ for _ in range(9):
+ version, status, reason = super()._read_status()
+ if not 100 <= status < 200 or status == 101:
+ return version, status, reason
+ http.client.parse_headers(self.fp)
+ raise http.client.HTTPException("too many informational responses")
+
+
+def origin(value: str) -> str:
+ p = urllib.parse.urlsplit(value)
+ if (p.scheme not in {"http", "https"} or not p.hostname
+ or p.username is not None or p.password is not None
+ or p.path not in {"", "/"} or p.query or p.fragment
+ or any(c.isspace() for c in value)):
+ raise ValueError("expected an http(s) origin without credentials, path, query or fragment")
+ _ = p.port # validate malformed/out-of-range ports before any request
+ return urllib.parse.urlunsplit((p.scheme, p.netloc, "", "", ""))
+
+
+def fetch(base: str, path: str, method: str, timeout: float):
+ p = urllib.parse.urlsplit(base)
+ cls = http.client.HTTPSConnection if p.scheme == "https" else http.client.HTTPConnection
+ conn = cls(p.hostname, p.port, timeout=timeout)
+ conn.response_class = FinalResponse
+ try:
+ conn.request(method, path, headers={"User-Agent": USER_AGENT,
+ "Accept-Encoding": "identity"})
+ response = conn.getresponse()
+ body = response.read(MAX_BYTES + 1)
+ if len(body) > MAX_BYTES:
+ raise ValueError("response exceeds the 1 MiB verification limit")
+ return response.status, response.headers, body, None
+ except (OSError, ValueError, http.client.HTTPException) as exc:
+ return 0, Message(), b"", f"{type(exc).__name__}: {exc}"
+ finally:
+ conn.close()
+
+
+def split_links(value: str) -> list[str]:
+ """Split commas outside URI brackets and quoted strings (RFC 8288)."""
+ parts, start, quoted, escaped, bracket = [], 0, False, False, False
+ for i, char in enumerate(value):
+ if escaped:
+ escaped = False
+ elif quoted and char == "\\":
+ escaped = True
+ elif char == '"' and not bracket:
+ quoted = not quoted
+ elif not quoted:
+ if char == "<":
+ bracket = True
+ elif char == ">":
+ bracket = False
+ elif char == "," and not bracket:
+ parts.append(value[start:i])
+ start = i + 1
+ if quoted or bracket:
+ raise ValueError("unclosed Link URI or quoted string")
+ return parts + [value[start:]]
+
+
+def relations(headers: Message, base: str) -> set[tuple[str, str]]:
+ found = set()
+ for value in headers.get_all("Link", []):
+ for link in split_links(value):
+ match = re.fullmatch(r"\s*<([^<>]*)>\s*(;.*)?\s*", link)
+ if not match:
+ raise ValueError("malformed Link field")
+ params = Message()
+ params["Content-Type"] = "link" + (match[2] or "")
+ # get_param takes the first rel; a later duplicate must not turn
+ # an unrelated link into a valid discovery relation (RFC 8288).
+ rel = params.get_param("rel", "")
+ if not isinstance(rel, str) or params.get_param("anchor") is not None:
+ continue # an explicit anchor changes the link's context
+ target = urllib.parse.urljoin(base + "/", match[1])
+ for name in rel.split():
+ found.add((name.lower(), target))
+ return found
+
+
+def verify(base: str, *, skills: bool = False, timeout: float = 15) -> list[dict]:
+ results = []
+
+ def request(path, method="GET", media=None):
+ status, headers, body, error = fetch(base, path, method, timeout)
+ failures = [error] if error else []
+ if not error and status != 200:
+ failures.append(f"HTTP {status}; expected 200 (redirects are not followed)")
+ if status == 200 and media and (len(headers.get_all("Content-Type", [])) != 1
+ or headers.get_content_type() != media):
+ failures.append(f"Content-Type {headers.get('Content-Type')!r}; expected {media}")
+ row = {"method": method, "url": base + path, "status": status,
+ "failures": failures}
+ results.append(row)
+ return row, headers, body
+
+ for method in ("GET", "HEAD"):
+ row, headers, _ = request("/", method, "text/html")
+ if row["status"] == 200:
+ try:
+ found = relations(headers, base)
+ for rel, (path, _) in TARGETS.items():
+ if (rel, base + path) not in found:
+ row["failures"].append(f"missing {rel} link to {path} on final response")
+ except ValueError as exc:
+ row["failures"].append(str(exc))
+
+ for rel, (path, media) in TARGETS.items():
+ row, _, body = request(path, media=media)
+ if rel == "service-desc" and not row["failures"]:
+ try:
+ spec = json.loads(body)
+ if not isinstance(spec, dict) or not isinstance(spec.get("openapi"), str) \
+ or not isinstance(spec.get("paths"), dict):
+ raise ValueError("missing OpenAPI version or paths object")
+ except (ValueError, UnicodeError) as exc:
+ row["failures"].append(f"invalid OpenAPI JSON: {exc}")
+
+ if skills:
+ row, headers, body = request(SKILL_INDEX, media="application/json")
+ for key, expected in (("Access-Control-Allow-Origin", "*"),):
+ if row["status"] == 200 and headers.get(key) != expected:
+ row["failures"].append(f"{key} must be {expected}")
+ digest = None
+ if not row["failures"]:
+ try:
+ index = json.loads(body)
+ if not isinstance(index, dict) or index.get("$schema") != SKILL_SCHEMA \
+ or not isinstance(index.get("skills"), list):
+ raise ValueError("expected the discovery v0.2.0 schema and skills array")
+ entries = [e for e in index["skills"] if isinstance(e, dict) and e.get("name") == "xerj"]
+ if len(entries) != 1:
+ raise ValueError("expected exactly one xerj entry")
+ entry = entries[0]
+ if (entry.get("type") != "skill-md"
+ or not isinstance(entry.get("url"), str)
+ or urllib.parse.urljoin(base + SKILL_INDEX, entry["url"]) != base + SKILL_FILE
+ or not isinstance(entry.get("description"), str)
+ or not 1 <= len(entry["description"].strip()) <= 1024
+ or not isinstance(entry.get("digest"), str)
+ or not re.fullmatch(r"sha256:[0-9a-f]{64}", entry["digest"])):
+ raise ValueError("invalid xerj skill metadata, URL or digest")
+ digest = entry["digest"]
+ except (ValueError, UnicodeError) as exc:
+ row["failures"].append(f"invalid skill index: {exc}")
+ if digest is not None:
+ row, headers, body = request(SKILL_FILE, media="text/markdown")
+ if row["status"] == 200:
+ if headers.get("Access-Control-Allow-Origin") != "*":
+ row["failures"].append("Access-Control-Allow-Origin must be *")
+ if "sha256:" + hashlib.sha256(body).hexdigest() != digest:
+ row["failures"].append("skill bytes do not match the index SHA-256")
+
+ for row in results:
+ row["ok"] = not row["failures"]
+ return results
+
+
+def main(argv=None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("base", nargs="?", default="https://xerj.org")
+ parser.add_argument("--skills", action="store_true", help="also require the public xerj skill index and digest")
+ parser.add_argument("--timeout", type=float, default=15, help="per socket operation, seconds (0 < n <= 60)")
+ parser.add_argument("--json", action="store_true")
+ args = parser.parse_args(argv)
+ try:
+ base = origin(args.base)
+ if not 0 < args.timeout <= 60:
+ raise ValueError("timeout must be greater than 0 and at most 60 seconds")
+ except ValueError as exc:
+ parser.error(str(exc))
+ results = verify(base, skills=args.skills, timeout=args.timeout)
+ failed = sum(not row["ok"] for row in results)
+ report = {"base": base, "skills": args.skills, "checked": len(results),
+ "failed": failed, "results": results}
+ if args.json:
+ print(json.dumps(report, indent=2))
+ else:
+ for row in results:
+ print(f"{'PASS' if row['ok'] else 'FAIL'} {row['method']} {row['url']} "
+ f"HTTP {row['status']} " + json.dumps(row["failures"], ensure_ascii=True))
+ print(f"verify_agent_discovery: {len(results)} requests, {failed} failed")
+ return 1 if failed else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())