An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
"""逐架构验证临时 Docker 镜像,并在验收通过后按 digest 发布。"""
import argparse
import json
from pathlib import Path
import re
import subprocess
import time
import unicodedata
import urllib.error
import urllib.request
import uuid
PLATFORMS = ("linux/amd64", "linux/arm64", "linux/arm/v7", "linux/arm/v8")
CHECKS = {"licenses", "version", "model-load", "chinese", "latin", "blank", "worker-failure", "notebook"}
DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
# 与 app/package.json 的版本一致:正式版为 x.y.z,预发行版追加 -alpha.1、-beta.2 或 -rc.3。
VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:-(?:alpha|beta|rc)(?:\.[0-9A-Za-z]+)*)?")
def run(arguments, *, input=None, timeout=300, check=True):
result = subprocess.run(arguments, input=input, text=True, encoding="utf-8", errors="replace",
capture_output=True, timeout=timeout)
if check and result.returncode:
raise RuntimeError(f"Command failed ({result.returncode}): {arguments[0]}\n{result.stderr}")
return result
def platform_name(platform):
name = platform.get("os", "") + "/" + platform.get("architecture", "")
variant = platform.get("variant", "")
if variant and not (name == "linux/arm64" and variant == "v8"):
name += "/" + variant
return name
def manifests(index, expected=PLATFORMS):
result = {}
for entry in index.get("manifests", []):
if entry.get("annotations", {}).get("vnd.docker.reference.type") == "attestation-manifest":
continue
name = platform_name(entry.get("platform", {}))
digest = entry.get("digest", "")
if name in result or not DIGEST.fullmatch(digest):
raise ValueError(f"Invalid or duplicate image manifest: {name}")
result[name] = digest
if set(result) != set(expected):
raise ValueError(f"Expected {expected}, found {tuple(result)}")
return result
def normalized(text):
return "".join(unicodedata.normalize("NFKC", text).split()).casefold()
def validate_rows(output, expected):
rows = json.loads(output)
if not isinstance(rows, list) or any(not isinstance(row, dict) or not isinstance(row.get("text"), str) for row in rows):
raise ValueError("Invalid OCR response")
if expected is None:
if rows:
raise ValueError("Blank image produced OCR text")
elif normalized(expected) not in normalized("".join(row["text"] for row in rows)):
raise ValueError(f"Expected OCR text {expected!r}, received {rows!r}")
class KernelAPIError(ValueError):
def __init__(self, endpoint, result):
super().__init__(f"Kernel API failed: {endpoint}: {result}")
self.result = result
def request(base, endpoint, token, payload=None):
data = None if payload is None else json.dumps(payload).encode("utf-8")
req = urllib.request.Request(base + endpoint, data=data,
headers={"Content-Type": "application/json", "Authorization": "Token " + token})
with urllib.request.urlopen(req, timeout=10) as response:
result = json.load(response)
if result.get("code") != 0:
raise KernelAPIError(endpoint, result)
return result["data"]
def wait_notebook_ready(base, token, notebook, timeout):
# 创建笔记本会异步重建索引;只轮询只读接口,不重试可能已提交的文档写入。
deadline = time.monotonic() + timeout
while True:
try:
request(base, "/api/filetree/getDocPaths", token, {"notebook": notebook, "ids": [notebook]})
return
except KernelAPIError as error:
data = error.result.get("data")
if not isinstance(data, dict) or data.get("reason") != "indexUnavailable":
raise
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError(f"Notebook index did not become ready: {notebook}; last response: {error}") from error
time.sleep(min(1, remaining))
def verify_notebook(image, platform, version, timeout, fail_ocr, diagnostics=None):
# 只使用本次测试创建的容器和卷,所有读写均发生在临时工作空间。
name = "siyuan-docker-smoke-" + uuid.uuid4().hex
volume, token = name + "-workspace", uuid.uuid4().hex
run(["docker", "volume", "create", volume])
try:
run(["docker", "run", "--rm", "-i", "--platform", platform, "--entrypoint", "/bin/sh",
"--mount", f"type=volume,src={volume},dst=/siyuan/workspace", image, "-ec",
"mkdir -p /siyuan/workspace/conf; cat > /siyuan/workspace/conf/conf.json"],
input=json.dumps({"api": {"token": token}, "ocr": {"auto": False}}))
run(["docker", "run", "-d", "--name", name, "--platform", platform,
"--mount", f"type=volume,src={volume},dst=/siyuan/workspace",
"-p", "127.0.0.1::6806", image, "serve", "--port=6806", "--safe-mode",
"--accessAuthCode=" + token])
bindings = json.loads(run(["docker", "inspect", name]).stdout)[0]["NetworkSettings"]["Ports"]["6806/tcp"]
base = "http://127.0.0.1:" + bindings[0]["HostPort"]
deadline = time.monotonic() + timeout
last_error = ""
while True:
try:
progress = request(base, "/api/system/bootProgress", token)
if progress["progress"] == 100:
break
except (urllib.error.URLError, TimeoutError, ConnectionError, ValueError) as error:
# 端口转发已建立但内核尚未就绪时,HTTP 请求可能被重置或提前断开。
last_error = str(error)
if time.monotonic() >= deadline:
raise TimeoutError(f"Kernel did not boot: {platform}; last response: {last_error}")
if run(["docker", "inspect", "--format", "{{.State.Running}}", name]).stdout.strip() != "true":
raise RuntimeError("Kernel stopped before booting")
time.sleep(2)
if request(base, "/api/system/version", token) != version:
raise ValueError("HTTP kernel version differs from release version")
fail_ocr()
notebook = request(base, "/api/notebook/createNotebook", token, {"name": "Docker OCR smoke"})
wait_notebook_ready(base, token, notebook["notebook"]["id"], timeout)
block = request(base, "/api/filetree/createDocByParent", token,
{"notebook": notebook["notebook"]["id"], "parentID": None,
"id": time.strftime("%Y%m%d%H%M%S") + "-" + uuid.uuid4().hex[:7],
"title": {"format": "plain", "text": "Docker smoke"}, "markdown": "Docker OCR smoke 123"})
data = request(base, "/api/block/getBlockKramdown", token, {"id": block["id"]})
if "Docker OCR smoke 123" not in data["kramdown"]:
raise ValueError("Notebook write/read verification failed")
except Exception:
# 删除临时容器前保留启动日志和退出状态,采集失败不能覆盖原始验收错误。
for filename, command in (("kernel.log", ["docker", "logs", name]),
("kernel-state.json", ["docker", "inspect", "--format", "{{json .State}}", name])):
try:
result = run(command, check=False, timeout=30)
output = result.stdout + result.stderr
print(f"{platform} {filename}:\n{output}", flush=True)
if diagnostics is not None:
diagnostics.mkdir(parents=True, exist_ok=True)
(diagnostics / filename).write_text(output, encoding="utf-8")
except Exception as error:
print(f"Unable to collect {filename}: {error}", flush=True)
raise
finally:
run(["docker", "rm", "-f", name], check=False)
run(["docker", "volume", "rm", volume], check=False)
def verify_platform(repository, digest, platform, version, fixtures, timeout, diagnostics=None):
started = time.monotonic()
image = repository + "@" + digest
architecture = platform.split("/")[1]
runtime = "/opt/siyuan/stage/ocr/runtime/linux-" + architecture
worker = runtime + "/siyuan-ocr"
prefix = ["docker", "run", "--rm", "--platform", platform]
checks = []
run(["docker", "pull", "--platform", platform, image], timeout=900)
# 辅助程序的动态链接由实际模型加载和识别验证,避免 musl 的 ldd 在 ARM32 模拟环境中误判。
shell = f"test -s LICENSE && test -s THIRD_PARTY_NOTICES.md && test -s {runtime}/LICENSE " \
f"&& test -s {runtime}/ThirdPartyNotices.txt && test -s {runtime}/onnx-toolchain.txt " \
f"&& test -x {worker} && ldd {runtime}/libonnxruntime.so"
run(prefix + ["--entrypoint", "/bin/sh", image, "-ec", shell])
checks.append("licenses")
output = run(prefix + ["--entrypoint", "/opt/siyuan/kernel", image, "--version"]).stdout
if not re.search(r"\bversion\s+" + re.escape(version) + r"(?![\w.-])", output):
raise ValueError(f"Unexpected kernel version: {output}")
checks.append("version")
worker_name = "siyuan-docker-ocr-" + uuid.uuid4().hex
prefix += ["--name", worker_name, "-i", "--mount", f"type=bind,src={fixtures.resolve()},dst=/fixtures,readonly",
"--entrypoint", worker, image]
config = {"library": runtime + "/libonnxruntime.so", "directory": "/opt/siyuan/stage/ocr/models/tiny"}
try:
for name, expected in (("model-load", None), ("chinese", "思源笔记"), ("latin", "SiYuan OCR 123"), ("blank", None)):
payload = {"config": config, "image": "" if name == "model-load" else f"/fixtures/{name}.png"}
result = run(prefix, input=json.dumps(payload), timeout=timeout)
validate_rows(result.stdout, expected)
checks.append(name)
# 模拟辅助进程加载失败,再验证已启动的内核仍可读写普通文档。
def fail_ocr():
bad = {"config": {**config, "library": runtime + "/missing-onnxruntime.so"}, "image": ""}
failure = run(prefix, input=json.dumps(bad), timeout=timeout, check=False)
if not failure.returncode or "missing-onnxruntime.so" not in failure.stderr:
raise ValueError("Missing OCR runtime did not produce the expected failure")
checks.append("worker-failure")
verify_notebook(image, platform, version, timeout, fail_ocr, diagnostics)
checks.append("notebook")
finally:
run(["docker", "rm", "-f", worker_name], check=False)
return {"platform": platform, "digest": digest, "checks": checks, "seconds": round(time.monotonic() - started, 3)}
def verify(image, version, fixtures, report_path, timeout, *, platform=None, provenance=None):
repository, separator, digest = image.partition("@")
if not separator or not DIGEST.fullmatch(digest):
raise ValueError("Verification requires an immutable image digest")
for name in ("chinese", "latin", "blank"):
if not (fixtures / f"{name}.png").is_file():
raise ValueError(f"Missing OCR fixture: {name}")
expected = (platform,) if platform else PLATFORMS
index = manifests(json.loads(run(["docker", "buildx", "imagetools", "inspect", image, "--raw"]).stdout), expected)
report = {"image": image, "version": version, "status": "running", "platforms": [], **(provenance or {})}
try:
for platform in expected:
print(f"Verifying {platform} in Docker", flush=True)
diagnostics = report_path.parent / "diagnostics" / platform.replace("/", "-")
report["platforms"].append(verify_platform(repository, index[platform], platform, version, fixtures, timeout,
diagnostics=diagnostics))
report["status"] = "passed"
except Exception as error:
report.update(status="failed", error=str(error))
raise
finally:
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def verified_platforms(report, image, version, expected=PLATFORMS):
if report.get("image") != image or report.get("version") != version or report.get("status") != "passed":
raise ValueError("Image promotion requires a matching successful verification report")
platforms = report.get("platforms", [])
if len(platforms) != len(expected) or {value.get("platform") for value in platforms} != set(expected):
raise ValueError("Verification did not cover every release platform")
for value in platforms:
if set(value.get("checks", [])) != CHECKS or not DIGEST.fullmatch(value.get("digest", "")):
raise ValueError("Incomplete platform verification")
repository, separator, digest = image.partition("@")
if not separator or not DIGEST.fullmatch(digest):
raise ValueError("Image promotion requires an immutable digest")
return {value["platform"]: value["digest"] for value in platforms}
def assemble(repository, tag, version, reports_path, report_path, provenance):
# 重试失败任务时可复用同次运行中已通过的报告,但不接受其他运行或其他提交的产物。
reports = sorted(reports_path.rglob("report.json"))
if len(reports) != len(PLATFORMS):
raise ValueError("Assembly requires exactly four platform reports")
expected, sources, entries = {}, [], []
for path in reports:
report = json.loads(path.read_text(encoding="utf-8"))
if any(report.get(key) != value for key, value in provenance.items()):
raise ValueError("Platform report belongs to another source, configuration or workflow run")
platform_entries = report.get("platforms", [])
if len(platform_entries) != 1 or platform_entries[0].get("platform") not in PLATFORMS:
raise ValueError("Expected one supported platform per report")
platform = platform_entries[0]["platform"]
image = report.get("image", "")
if image.partition("@")[0] != repository or platform in expected:
raise ValueError("Unexpected repository or duplicate platform report")
verified = verified_platforms(report, image, version, (platform,))
actual = manifests(json.loads(run(["docker", "buildx", "imagetools", "inspect", image, "--raw"]).stdout), (platform,))
if actual != verified:
raise ValueError("Platform report digest differs from candidate image")
expected.update(verified)
sources.append(image)
entries.extend(platform_entries)
if set(expected) != set(PLATFORMS):
raise ValueError("Assembly is missing a release platform")
candidate = repository + ":" + tag
run(["docker", "buildx", "imagetools", "create", "--tag", candidate, *sources], timeout=900)
descriptor = json.loads(run(["docker", "buildx", "imagetools", "inspect", candidate,
"--format", "{{json .Manifest}}"]).stdout)
digest = descriptor.get("digest", "")
if not DIGEST.fullmatch(digest):
raise ValueError("Missing assembled image digest")
image = repository + "@" + digest
actual = manifests(json.loads(run(["docker", "buildx", "imagetools", "inspect", image, "--raw"]).stdout))
if actual != expected:
raise ValueError("Assembled manifest differs from the verified platform images")
report = {"image": image, "version": version, "status": "passed", "platforms": entries, **provenance}
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def release_tags(version):
"""预发行版只发布版本标签,latest 始终指向正式版。"""
tags = [":v" + version]
if "-" not in version:
tags.append(":latest")
return tags
def promote(image, version, report_path):
report = json.loads(report_path.read_text(encoding="utf-8"))
expected = verified_platforms(report, image, version)
repository = image.partition("@")[0]
actual = manifests(json.loads(run(["docker", "buildx", "imagetools", "inspect", image, "--raw"]).stdout))
if actual != expected:
raise ValueError("Candidate manifest differs from the verified image")
tags = release_tags(version)
command = ["docker", "buildx", "imagetools", "create"]
for tag in tags:
command += ["--tag", repository + tag]
run(command + [image], timeout=900)
for tag in tags:
published = manifests(json.loads(run(["docker", "buildx", "imagetools", "inspect", repository + tag, "--raw"]).stdout))
if published != expected:
raise ValueError(f"Published manifest differs from the verified image: {tag}")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("command", choices=("verify", "assemble", "promote"))
parser.add_argument("--image")
parser.add_argument("--version", required=True)
parser.add_argument("--report", type=Path, required=True)
parser.add_argument("--fixtures", type=Path, default=Path(__file__).parent / "testdata/docker-ocr")
parser.add_argument("--timeout", type=int, default=240)
parser.add_argument("--platform", choices=PLATFORMS)
parser.add_argument("--repository")
parser.add_argument("--candidate-tag")
parser.add_argument("--reports", type=Path)
parser.add_argument("--source-commit")
parser.add_argument("--configuration-commit")
parser.add_argument("--run-id")
args = parser.parse_args()
if not VERSION.fullmatch(args.version) or args.timeout <= 0:
parser.error("Expected a release version and positive timeout")
provenance = {"source_commit": args.source_commit, "configuration_commit": args.configuration_commit,
"run_id": args.run_id}
if args.command == "assemble" or args.platform:
if not all(re.fullmatch(r"[0-9a-f]{40}", value or "")
for value in (args.source_commit, args.configuration_commit)) or not re.fullmatch(r"[0-9]+", args.run_id or ""):
parser.error("Platform verification requires pinned source/configuration commits and a workflow run ID")
if args.command != "assemble" and not args.image:
parser.error("Expected an immutable --image")
if args.command == "verify":
verify(args.image, args.version, args.fixtures, args.report, args.timeout,
platform=args.platform, provenance=provenance if args.platform else None)
elif args.command == "assemble":
if not re.fullmatch(r"[a-z0-9][a-z0-9./_-]*", args.repository or "") \
or not re.fullmatch(r"build-[0-9]+-[0-9]+-verified", args.candidate_tag or "") or not args.reports:
parser.error("Expected a repository, reports directory and temporary build candidate tag")
assemble(args.repository, args.candidate_tag, args.version, args.reports, args.report, provenance)
else:
promote(args.image, args.version, args.report)
if __name__ == "__main__":
main()