Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

fix(installer): recover interrupted HTTPS archive downloads (#95)

澄潭257ad95897c7
diff --git a/docs/standalone-cli.md b/docs/standalone-cli.md
index 3eac8dcb..0e182ca6 100644
--- a/docs/standalone-cli.md
+++ b/docs/standalone-cli.md
@@ -59,9 +59,14 @@ Use the component version and full source revision of the same committed candida
After distribution integration, the installer also accepts
`--manifest-url https://.../PACKAGE.tar.gz.json`. Redirects and archives must remain HTTPS.
-Before any write, the installer checks the platform, archive SHA-256, closed file set, and
+Before changing the installation, the installer checks the platform, archive SHA-256, closed file set, and
per-file digests. Never infer an unpublished download URL from this page.
+HTTPS archive downloads use bounded requests and retries when a transfer ends early.
+When combining byte ranges, the installer requires one unchanged strong object identity;
+servers without range support retain whole-file downloads. All size and digest checks
+still complete before changing an existing installation.
+
The collector uses only locked Rust dependencies and the CPA commit pinned by
`vendor/cpa/source.json`. `--cpa-source-repo` supplies local objects for that exact commit and
does not use the checkout's current branch. The packager requires the collector's closed
diff --git a/docs/standalone-cli.zh-CN.md b/docs/standalone-cli.zh-CN.md
index 946f92b5..817df74e 100644
--- a/docs/standalone-cli.zh-CN.md
+++ b/docs/standalone-cli.zh-CN.md
@@ -54,9 +54,13 @@ python3 scripts/install-standalone.py install \
完整源码 revision。
集成后安装器也接受 `--manifest-url https://.../PACKAGE.tar.gz.json`。重定向和归档必须保持
-HTTPS;安装器会在任何写入前校验平台、归档 SHA-256、闭合集合和逐文件摘要。不要根据本页
+HTTPS;安装器会在变更安装前校验平台、归档 SHA-256、闭合集合和逐文件摘要。不要根据本页
拼造未发布的下载地址。
+HTTPS 归档下载会使用有大小边界的请求,并对提前结束的传输进行有限次重试。
+合并分段数据时,安装器要求服务端对象身份保持不变;不支持分段的服务端仍可整包下载。
+所有大小和摘要检查通过前,安装器不会更改现有安装。
+
收集器只从锁定的 Rust 依赖和 `vendor/cpa/source.json` 指定的 CPA commit 生成材料;
`--cpa-source-repo` 只是该固定 commit 的本地对象来源,不会采用 checkout 当前分支。
打包器要求收集器的闭合输出,并将 HiRoute、CPA 及其传递依赖的许可证和确定性清单随归档
diff --git a/scripts/install-standalone.py b/scripts/install-standalone.py
index c80ef9d3..e5e8ed27 100755
--- a/scripts/install-standalone.py
+++ b/scripts/install-standalone.py
@@ -5,6 +5,7 @@ import argparse
from contextlib import contextmanager
import fcntl
import hashlib
+import http.client
import json
import os
from pathlib import Path
@@ -17,6 +18,7 @@ import subprocess
import sys
import tarfile
import tempfile
+import urllib.error
import urllib.parse
import urllib.request
@@ -28,6 +30,8 @@ MANAGED_TEXT = "Managed by HiRoute standalone installer"
SERVICE_LABEL = "ai.hiroute.cli"
MAX_MANIFEST = 1024 * 1024
MAX_ARCHIVE = 1024 * 1024 * 1024
+DOWNLOAD_CHUNK_SIZE = 8 * 1024 * 1024
+DOWNLOAD_ATTEMPTS = 3
PORTABLE = re.compile(r"^[A-Za-z0-9._-]+$")
HEX_SHA256 = re.compile(r"^[a-fA-F0-9]{64}$")
EXPECTED_FILES = {
@@ -108,24 +112,103 @@ def layout(home):
}
-def fetch_https(url, destination, limit):
+class IncompleteDownload(OSError):
+ pass
+
+
+def https_response(url, headers):
+ request = urllib.request.Request(url, headers={"User-Agent": "HiRoute-Installer/1", **headers})
+ try:
+ response = urllib.request.urlopen(request, timeout=30)
+ except urllib.error.HTTPError:
+ raise
+ except (urllib.error.URLError, OSError) as error:
+ raise IncompleteDownload("HTTPS connection interrupted") from error
+ final = urllib.parse.urlparse(response.geturl())
+ if final.scheme != "https" or not final.netloc or final.username or final.password:
+ response.close()
+ raise ValueError("download redirect left HTTPS")
+ return response
+
+
+def https_chunks(response, limit):
+ total = 0
+ while True:
+ try:
+ chunk = response.read(1024 * 1024)
+ except (http.client.HTTPException, OSError) as error:
+ raise IncompleteDownload("HTTPS transfer interrupted") from error
+ if not chunk:
+ break
+ total += len(chunk)
+ if total > limit:
+ raise ValueError("download exceeds the bounded size")
+ yield chunk
+ declared = response.headers.get("Content-Length")
+ if declared is not None and total != int(declared):
+ raise IncompleteDownload("HTTPS transfer ended before Content-Length")
+
+
+def fetch_whole_https(url, destination, limit, expected_size=None):
+ for attempt in range(DOWNLOAD_ATTEMPTS):
+ try:
+ with https_response(url, {}) as response, destination.open("wb") as output:
+ total = 0
+ for chunk in https_chunks(response, limit):
+ output.write(chunk)
+ total += len(chunk)
+ if expected_size is not None and total != expected_size:
+ raise IncompleteDownload("HTTPS transfer ended before the manifest size")
+ return
+ except IncompleteDownload:
+ if attempt + 1 == DOWNLOAD_ATTEMPTS:
+ raise
+
+
+def fetch_https(url, destination, limit, expected_size=None):
parsed = urllib.parse.urlparse(url)
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
raise ValueError("download URL must be credential-free HTTPS")
- request = urllib.request.Request(url, headers={"User-Agent": "HiRoute-Installer/1"})
- with urllib.request.urlopen(request, timeout=30) as response, destination.open("wb") as output:
- final = urllib.parse.urlparse(response.geturl())
- if final.scheme != "https" or not final.netloc or final.username or final.password:
- raise ValueError("download redirect left HTTPS")
- total = 0
- while True:
- chunk = response.read(1024 * 1024)
- if not chunk:
+ if expected_size is None:
+ return fetch_whole_https(url, destination, limit)
+ if type(expected_size) is not int or not 0 < expected_size <= limit:
+ raise ValueError("archive size exceeds the bounded size")
+ destination.write_bytes(b"")
+ offset, etag = 0, None
+ while offset < expected_size:
+ end = min(offset + DOWNLOAD_CHUNK_SIZE, expected_size) - 1
+ headers = {"Range": f"bytes={offset}-{end}"}
+ if etag is not None:
+ headers["If-Match"] = etag
+ for attempt in range(DOWNLOAD_ATTEMPTS):
+ try:
+ with https_response(url, headers) as response:
+ if response.status == 200 and offset == 0:
+ response.close()
+ return fetch_whole_https(url, destination, expected_size, expected_size)
+ if (
+ response.status != 206
+ or response.headers.get("Content-Range") != f"bytes {offset}-{end}/{expected_size}"
+ ):
+ raise ValueError("download returned an invalid byte range")
+ identity = response.headers.get("ETag")
+ if etag is not None and identity != etag:
+ raise ValueError("download object identity changed")
+ if offset == 0 and end + 1 < expected_size and (
+ identity is None or not re.fullmatch(r'"[\x21\x23-\x7e\x80-\xff]*"', identity)
+ ):
+ response.close()
+ return fetch_whole_https(url, destination, expected_size, expected_size)
+ value = b"".join(https_chunks(response, end - offset + 1))
+ if len(value) != end - offset + 1:
+ raise IncompleteDownload("HTTPS byte range ended early")
break
- total += len(chunk)
- if total > limit:
- raise ValueError("download exceeds the bounded size")
- output.write(chunk)
+ except IncompleteDownload:
+ if attempt + 1 == DOWNLOAD_ATTEMPTS:
+ raise
+ with destination.open("ab") as output:
+ output.write(value)
+ etag, offset = identity, end + 1
def acquire(args, temporary):
@@ -138,7 +221,7 @@ def acquire(args, temporary):
validate_manifest_shape(manifest)
archive_path = temporary / manifest["archive"]["filename"]
archive_url = urllib.parse.urljoin(args.manifest_url, manifest["archive"]["filename"])
- fetch_https(archive_url, archive_path, MAX_ARCHIVE)
+ fetch_https(archive_url, archive_path, MAX_ARCHIVE, manifest["archive"]["size"])
else:
if args.archive is None:
raise ValueError("a local manifest requires --archive")
diff --git a/scripts/test-standalone-install.py b/scripts/test-standalone-install.py
index ef4df22f..7bb8be51 100755
--- a/scripts/test-standalone-install.py
+++ b/scripts/test-standalone-install.py
@@ -3,10 +3,12 @@
import argparse
import importlib.util
+import io
import json
import os
from pathlib import Path
import plistlib
+import re
import sys
import tempfile
import unittest
@@ -26,6 +28,17 @@ package = load("package_standalone", "package-standalone.py")
installer = load("install_standalone", "install-standalone.py")
+class HTTPSResponse(io.BytesIO):
+ def __init__(self, body, status=200, headers=None, url="https://downloads.example/package"):
+ super().__init__(body)
+ self.status = status
+ self.headers = {"Content-Length": str(len(body)), **(headers or {})}
+ self.url = url
+
+ def geturl(self):
+ return self.url
+
+
class StandaloneFixture(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
@@ -412,6 +425,108 @@ class InstallerTests(StandaloneFixture):
installer.install(self.install_args(manifest, archive))
self.assertFalse((self.home / ".local").exists())
+ def https_package(self, manifest, archive, mode):
+ payload = archive.read_bytes()
+ calls = []
+
+ def response(request, **_options):
+ if request.full_url.endswith(manifest.name):
+ return HTTPSResponse(manifest.read_bytes())
+ self.assertTrue(request.full_url.endswith(archive.name))
+ value = request.get_header("Range")
+ calls.append((value, request.get_header("If-match")))
+ if not value or mode == "ignore-ranges":
+ body = payload[:len(payload) // 2] if mode in ("short-once", "always-short") else payload
+ return HTTPSResponse(body, headers={"Content-Length": str(len(payload))})
+ match = re.fullmatch(r"bytes=(\d+)-(\d+)", value)
+ self.assertIsNotNone(match)
+ start, end = map(int, match.groups())
+ body = payload[start:end + 1]
+ headers = {"Content-Range": f"bytes {start}-{end}/{len(payload)}", "ETag": '"fixture-object"'}
+ if mode == "short-once" and len(calls) == 1 or mode == "always-short":
+ headers["Content-Length"] = str(len(body))
+ body = body[:len(body) // 2]
+ if mode == "changed-object" and start > 0:
+ headers["ETag"] = '"different-object"'
+ if mode == "invalid-range" and start > 0:
+ headers["Content-Range"] = f"bytes {start}-{end}/{len(payload) + 1}"
+ if mode == "weak-etag":
+ headers["ETag"] = 'W/"fixture-object"'
+ url = "http://downloads.example/package" if mode == "unsafe-redirect" else "https://downloads.example/package"
+ return HTTPSResponse(body, 206, headers, url)
+
+ args = argparse.Namespace(manifest=None, archive=None,
+ manifest_url="https://downloads.example/" + manifest.name)
+ return args, response, calls
+
+ def test_https_partial_transfer_recovers_through_verified_install(self):
+ manifest, archive = self.build()
+ args, response, calls = self.https_package(manifest, archive, "short-once")
+ with self.environment(), patch.object(installer, "DOWNLOAD_CHUNK_SIZE", 128, create=True), \
+ patch.object(installer.urllib.request, "urlopen", side_effect=response), \
+ patch.object(installer.subprocess, "run"), patch("builtins.print"):
+ installer.install(args)
+ marker = json.loads((self.home / ".local/share/hiroute/standalone.json").read_text())
+ self.assertEqual(marker["version"], json.loads(manifest.read_text())["version"])
+ self.assertEqual((Path(marker["install_root"]) / "hiroute").read_bytes(), self.sources["hiroute"].read_bytes())
+ self.assertGreater(len(calls), 2)
+ self.assertEqual(calls[0][0], calls[1][0])
+ self.assertTrue(all(identity == '"fixture-object"' for _, identity in calls[2:]))
+
+ def test_https_changed_object_or_range_preserves_installed_state(self):
+ manifest, archive = self.build()
+ with self.environment(), patch.object(installer.subprocess, "run"), patch("builtins.print"):
+ installer.install(self.install_args(manifest, archive))
+ marker = self.home / ".local/share/hiroute/standalone.json"
+ original = marker.read_bytes()
+ user_data = self.home / ".local/share/hiroute/user-data"
+ user_data.write_bytes(b"preserve user data")
+ manifest, archive = self.build("0.2.0")
+ for mode, message in [("changed-object", "object identity"), ("invalid-range", "byte range")]:
+ args, response, _ = self.https_package(manifest, archive, mode)
+ with self.subTest(mode=mode), self.environment(), \
+ patch.object(installer, "DOWNLOAD_CHUNK_SIZE", 128, create=True), \
+ patch.object(installer.urllib.request, "urlopen", side_effect=response), \
+ patch.object(installer.subprocess, "run"), patch("builtins.print"):
+ with self.assertRaisesRegex(ValueError, message):
+ installer.install(args)
+ self.assertEqual(marker.read_bytes(), original)
+ self.assertEqual(user_data.read_bytes(), b"preserve user data")
+ self.assertFalse((self.home / ".local/lib/hiroute/0.2.0").exists())
+
+ def test_https_repeated_short_transfer_is_bounded_and_makes_no_install_writes(self):
+ manifest, archive = self.build()
+ args, response, calls = self.https_package(manifest, archive, "always-short")
+ with self.environment(), patch.object(installer, "DOWNLOAD_CHUNK_SIZE", 128, create=True), \
+ patch.object(installer.urllib.request, "urlopen", side_effect=response), \
+ patch.object(installer.subprocess, "run"), patch("builtins.print"):
+ with self.assertRaisesRegex(OSError, "ended"):
+ installer.install(args)
+ self.assertEqual(len(calls), 3)
+ self.assertFalse((self.home / ".local").exists())
+
+ def test_https_whole_file_fallback_retains_verified_install(self):
+ manifest, archive = self.build()
+ for mode in ("ignore-ranges", "weak-etag"):
+ args, response, calls = self.https_package(manifest, archive, mode)
+ with self.subTest(mode=mode), self.environment(), \
+ patch.object(installer, "DOWNLOAD_CHUNK_SIZE", 128, create=True), \
+ patch.object(installer.urllib.request, "urlopen", side_effect=response), \
+ patch.object(installer.subprocess, "run"), patch("builtins.print"):
+ installer.install(args)
+ self.assertIsNone(calls[-1][0])
+ self.assertEqual((self.home / ".local/bin/hiroute").read_bytes(), self.sources["hiroute"].read_bytes())
+
+ def test_https_archive_redirect_cannot_leave_https(self):
+ manifest, archive = self.build()
+ args, response, _ = self.https_package(manifest, archive, "unsafe-redirect")
+ with self.environment(), patch.object(installer, "DOWNLOAD_CHUNK_SIZE", 128, create=True), \
+ patch.object(installer.urllib.request, "urlopen", side_effect=response), \
+ patch.object(installer.subprocess, "run"), patch("builtins.print"):
+ with self.assertRaisesRegex(ValueError, "left HTTPS"):
+ installer.install(args)
+ self.assertFalse((self.home / ".local").exists())
+
def test_manifest_path_escape_is_rejected_and_cross_version_update_converges(self):
manifest, archive = self.build()
value = json.loads(manifest.read_text())