Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Downloads: every ranged answer is checked, a file with a wrong SHA-256 is repaired piece by piece instead of deleted

Niko122163214a565580
diff --git a/setup.py b/setup.py
index 8d9f89c5a..723d84141 100644
--- a/setup.py
+++ b/setup.py
@@ -63,6 +63,7 @@ import zipfile
from pathlib import Path
from strata_control import acquisition
+from strata_control import atomic
from strata_control import draft_layer
from strata_control.discovery import in_trash
from strata_control import user_args
@@ -1975,6 +1976,63 @@ class _PieceFailed(Exception):
"""A piece failed DL_CHUNK_TRIES times."""
+class _Changed(Exception):
+ """The server's file is not the one the first pieces came from (another ETag): what was written cannot be trusted."""
+
+
+_CONTENT_RANGE = re.compile(r"\s*bytes (\d+)-(\d+)/(\d+)\s*$")
+
+
+def _etag(headers) -> str | None:
+ """The object identity a response names: ETag without the weak mark and quotes; None when it sends none."""
+ raw = (headers.get("ETag") or "").strip()
+ if raw[:2] in ("W/", "w/"):
+ raw = raw[2:]
+ return raw.strip('"') or None
+
+
+def _check_ranged(r, pos: int, end: int, total: int, etag: str | None):
+ """A ranged answer is trusted only when it is exactly the bytes asked for: status 206, Content-Range `pos-end/total`
+ (a 200 with the whole file, another range or another total is the server not honouring the request), a Content-Length
+ that fits, and the same object (ETag) as the first answer. _NoRange: this server cannot be used with ranges;
+ OSError: a bad answer worth asking again (another edge)."""
+ if r.status != 206:
+ raise _NoRange(f"HTTP {r.status}")
+ m = _CONTENT_RANGE.match(r.headers.get("Content-Range") or "")
+ if not m or (int(m.group(1)), int(m.group(2)), int(m.group(3))) != (pos, end, total):
+ raise _NoRange(f"Content-Range {(r.headers.get('Content-Range') or '')!r} for bytes {pos}-{end}/{total}")
+ cl = (r.headers.get("Content-Length") or "").strip()
+ if cl and cl.isdigit() and int(cl) != end - pos + 1:
+ raise OSError(f"Content-Length {cl} for {end - pos + 1} bytes asked")
+ got = _etag(r.headers)
+ if etag and got and got != etag:
+ raise OSError("the server answered with another version of the file")
+
+
+def _write_all(f, pos: int, data):
+ """Write `data` at `pos`; a raw unbuffered file may take fewer bytes than it was given, so go on until all are down."""
+ view = memoryview(data)
+ f.seek(pos)
+ done_n = 0
+ while done_n < len(view):
+ n = f.write(view[done_n:])
+ if not n:
+ raise OSError("the disk took no more bytes")
+ done_n += n
+
+
+def _read_exact(f, pos: int, n: int) -> bytes:
+ f.seek(pos)
+ parts = []
+ while n > 0:
+ b = f.read(n)
+ if not b:
+ break
+ parts.append(b)
+ n -= len(b)
+ return b"".join(parts)
+
+
def _url_ttl(url: str):
"""Seconds a signed URL (the CDN's, after the redirect) is still valid by its own query, else None."""
q = urllib.parse.parse_qs(urllib.parse.urlsplit(url).query)
@@ -1998,6 +2056,7 @@ class _Source:
def __init__(self, url: str, total: int):
self.url, self.total = url, total
self.final, self.deadline = None, None
+ self.etag = None # the object the first answer named; every later answer must match
self.lock = threading.Lock()
def resolve(self, stale=None) -> str:
@@ -2007,8 +2066,12 @@ class _Source:
req = urllib.request.Request(self.url, headers={"User-Agent": "strata-setup", "Range": "bytes=0-0"})
with _urlopen(req, timeout=60) as r:
cr = (r.headers.get("Content-Range") or "").strip()
- if r.status != 206 or cr not in (f"bytes 0-0/{self.total}", "bytes 0-0/*"):
+ if r.status != 206 or cr != f"bytes 0-0/{self.total}":
raise _NoRange(f"HTTP {r.status}, Content-Range {cr!r}")
+ tag = _etag(r.headers)
+ if self.etag and tag and tag != self.etag:
+ raise _Changed(f"the file on the server changed while it was downloaded ({self.etag[:12]} -> {tag[:12]})")
+ self.etag = self.etag or tag
r.read(1)
final = r.geturl() or self.url
ttl = _url_ttl(final)
@@ -2100,17 +2163,45 @@ def _load_pieces(part: Path, total: int) -> set:
return set(range(n)) if size >= total else set(range(size // DL_CHUNK))
-def _save_pieces(part: Path, total: int, done: set):
- """Flush the data first, then record the pieces: a list never names bytes the disk does not have."""
+def _save_pieces(part: Path, total: int, done: set) -> bool:
+ """Flush the data first, then record the pieces: a list never names bytes the disk does not have. False when the
+ list could not be written (an older list stays, which only costs re-downloading a few pieces after a stop)."""
try:
with open(part, "r+b") as g:
os.fsync(g.fileno())
side = part.with_name(part.name + ".chunks")
tmp = side.with_name(side.name + ".tmp")
tmp.write_text(json.dumps({"v": 1, "total": total, "chunk": DL_CHUNK, "done": _ranges(done)}), encoding="utf-8")
- os.replace(tmp, side)
+ with open(tmp, "r+b") as g:
+ os.fsync(g.fileno())
+ atomic.replace(tmp, side)
+ return True
except OSError:
- pass # only costs re-downloading a few pieces after a stop
+ return False
+
+
+def _claim_part(part: Path, url: str, etag: str | None = None):
+ """Record where a .part's bytes come from (<name>.part.src: the address asked, and the file's ETag when the server
+ names one). A .part written from another address (the other mirror, another revision) or from another version of
+ the file is dropped with its piece list: two sources are not mixed inside one file, even when they should hold the
+ same bytes. A .part from an older setup has no record and is kept as before."""
+ src = part.with_name(part.name + ".src")
+ try:
+ old = json.loads(src.read_text(encoding="utf-8")) if src.is_file() else None
+ except (OSError, ValueError):
+ old = {}
+ if old is not None:
+ same = isinstance(old, dict) and old.get("url") == url and not (old.get("etag") and etag and old.get("etag") != etag)
+ if not same:
+ part.unlink(missing_ok=True)
+ part.with_name(part.name + ".chunks").unlink(missing_ok=True)
+ src.unlink(missing_ok=True)
+ old = None
+ if old is None or (etag and not old.get("etag")):
+ try:
+ src.write_text(json.dumps({"url": url, "etag": etag}), encoding="utf-8")
+ except OSError:
+ pass
def _single_ready(part: Path):
@@ -2150,7 +2241,8 @@ def _take_run(st: _Pieces, todo: list) -> list:
def _fetch_run(src: _Source, st: _Pieces, f, run: list):
"""One ranged request for the neighbouring pieces in `run`, written at their offsets; each piece counts as finished
- when its last byte is down. A connection that breaks continues from the byte it reached, not from the piece start."""
+ when its last byte is down. Every answer is checked to be exactly the range asked for (status, Content-Range, length,
+ ETag); a connection that breaks continues from the byte it reached."""
start = run[0] * DL_CHUNK
end = min((run[-1] + 1) * DL_CHUNK, st.total) - 1
pos, tries = start, 0
@@ -2160,13 +2252,13 @@ def _fetch_run(src: _Source, st: _Pieces, f, run: list):
while True:
if st.stop.is_set():
return
- url, err, got, wait, ra = None, None, False, None, 0.0
+ url, err, wait, ra = None, None, None, 0.0
+ attempt_pos = pos
try:
url = src.get()
req = urllib.request.Request(url, headers={"User-Agent": "strata-setup", "Range": f"bytes={pos}-{end}"})
with _urlopen(req, timeout=60) as r:
- if r.status != 206 or not (r.headers.get("Content-Range") or "").strip().startswith(f"bytes {pos}-{end}/"):
- raise _NoRange(f"HTTP {r.status}")
+ _check_ranged(r, pos, end, st.total, src.etag)
while pos <= end:
if st.stop.is_set():
return
@@ -2176,12 +2268,10 @@ def _fetch_run(src: _Source, st: _Pieces, f, run: list):
if not b:
raise IOError("the connection closed early")
try:
- f.seek(pos)
- f.write(b)
+ _write_all(f, pos, b)
except OSError as e:
raise _DiskError(str(e)) from e
pos += len(b)
- got = True
st.add(cur, len(b))
if pos == cur_end:
st.finish(cur)
@@ -2192,7 +2282,7 @@ def _fetch_run(src: _Source, st: _Pieces, f, run: list):
st.flight.pop(i, None)
return
return
- except (_NoRange, _DiskError):
+ except (_NoRange, _DiskError, _Changed):
raise
except urllib.error.HTTPError as e:
err = e
@@ -2200,7 +2290,7 @@ def _fetch_run(src: _Source, st: _Pieces, f, run: list):
try:
src.resolve(stale=url)
wait = 0
- except _NoRange:
+ except (_NoRange, _Changed):
raise
except (OSError, http.client.HTTPException):
pass
@@ -2211,7 +2301,11 @@ def _fetch_run(src: _Source, st: _Pieces, f, run: list):
pass
except (OSError, http.client.HTTPException) as e:
err = e
- tries = 1 if got else tries + 1 # a try that moved forward is not a failed one
+ # A try that moved forward is not a failed one. The request goes on from the byte it reached: what came before the
+ # break is exactly the range that was asked for (the answer's status, Content-Range, length and ETag were checked and
+ # every byte was written in full), and writing a piece again from its start would make a route that resets more often
+ # than every DL_CHUNK bytes unable to finish any piece. The whole-file SHA-256 and the repair are the net behind it.
+ tries = 1 if pos > attempt_pos else tries + 1
if tries >= DL_CHUNK_TRIES:
raise _PieceFailed(str(err))
st.stop.wait(max(DL_BACKOFF * 2 ** (tries - 1), ra) if wait is None else wait)
@@ -2228,7 +2322,7 @@ def _piece_worker(src: _Source, st: _Pieces, todo: list, part: Path):
except _PieceFailed as e:
st.errors.append(e)
st.abort_round.set()
- except (_NoRange, _DiskError) as e:
+ except (_NoRange, _DiskError, _Changed) as e:
st.fatal = st.fatal or e
st.stop.set()
except OSError as e: # the .part itself could not be opened
@@ -2246,11 +2340,14 @@ def parallel_download(url: str, part: Path, total: int, label: str, conns: int,
return False
except (OSError, http.client.HTTPException):
return False # the one-stream path has its own retries and messages
+ part.parent.mkdir(parents=True, exist_ok=True)
+ _claim_part(part, url, src.etag) # a .part from another address or version is dropped here
done = _load_pieces(part, total)
st = _Pieces(total, done, conns)
- part.parent.mkdir(parents=True, exist_ok=True)
if not part.exists():
part.touch()
+ if not part.with_name(part.name + ".chunks").is_file() and not _save_pieces(part, total, done):
+ return False # no piece list can be kept here: the pieces would not be told from holes
last_print = last_save = 0.0
def report(force=False):
@@ -2292,6 +2389,12 @@ def parallel_download(url: str, part: Path, total: int, label: str, conns: int,
raise
report(force=True)
print()
+ if isinstance(st.fatal, _Changed):
+ part.unlink(missing_ok=True)
+ part.with_name(part.name + ".chunks").unlink(missing_ok=True)
+ part.with_name(part.name + ".src").unlink(missing_ok=True)
+ fail(f"{name} changed on the server while it was downloading: the part that was there was deleted",
+ "run it again (the download starts over with the file that is there now)")
if isinstance(st.fatal, _DiskError):
_save_pieces(part, total, st.snapshot())
fail(f"cannot write {part.name}: {st.fatal}", "free some disk space (or pick another --models-dir) and run it again")
@@ -2313,37 +2416,163 @@ def parallel_download(url: str, part: Path, total: int, label: str, conns: int,
return True
+def _repair_piece(src: _Source, st: _Pieces, f, i: int, stats: dict):
+ """Read piece `i` from the server in 1 MiB blocks and compare each with what is on the disk; a block that differs is
+ written again. A broken connection asks again from the block it reached (every block is final once compared)."""
+ pos, end = i * DL_CHUNK, min((i + 1) * DL_CHUNK, st.total) - 1
+ tries = 0
+ while pos <= end:
+ if st.stop.is_set():
+ return
+ url, wait, ra = None, None, 0.0
+ before = pos
+ try:
+ url = src.get()
+ req = urllib.request.Request(url, headers={"User-Agent": "strata-setup", "Range": f"bytes={pos}-{end}"})
+ with _urlopen(req, timeout=60) as r:
+ _check_ranged(r, pos, end, st.total, src.etag)
+ while pos <= end:
+ if st.stop.is_set():
+ return
+ b = r.read(min(1 << 20, end + 1 - pos))
+ if not b:
+ raise IOError("the connection closed early")
+ try:
+ if _read_exact(f, pos, len(b)) != b:
+ _write_all(f, pos, b)
+ with st.lock:
+ stats["bytes"] += len(b)
+ stats["pieces"].add(i)
+ except OSError as e:
+ raise _DiskError(str(e)) from e
+ pos += len(b)
+ st.add(i, len(b))
+ return
+ except (_NoRange, _DiskError, _Changed):
+ raise
+ except urllib.error.HTTPError as e:
+ if e.code in (401, 403, 410) and url is not None:
+ try:
+ src.resolve(stale=url)
+ wait = 0
+ except (_NoRange, _Changed):
+ raise
+ except (OSError, http.client.HTTPException):
+ pass
+ elif e.code in (429, 503):
+ try:
+ ra = min(float(e.headers.get("Retry-After") or 0), 60)
+ except (TypeError, ValueError):
+ pass
+ err = e
+ except (OSError, http.client.HTTPException) as e:
+ err = e
+ tries = 1 if pos > before else tries + 1
+ if tries >= DL_CHUNK_TRIES:
+ raise _PieceFailed(str(err))
+ st.stop.wait(max(DL_BACKOFF * 2 ** (tries - 1), ra) if wait is None else wait)
+
+
+def repair_download(url: str, path: Path, total: int, label: str, conns: int):
+ """A finished file whose SHA-256 is wrong is not thrown away: read it again from the server piece by piece, compare
+ each block with the disk and write only the blocks that differ. Returns (damaged pieces, pieces), or None when the
+ server cannot do it (no ranges, or the file there is another one); the caller checks the SHA-256 again."""
+ if not total or path.stat().st_size != total:
+ return None
+ src = _Source(url, total)
+ try:
+ src.resolve()
+ except (_NoRange, _Changed, OSError, http.client.HTTPException):
+ return None
+ st = _Pieces(total, set(), conns)
+ stats = {"bytes": 0, "pieces": set()}
+ todo = list(range(st.n))
+ errors = []
+
+ def worker():
+ try:
+ with open(path, "r+b", buffering=0) as f:
+ while not st.stop.is_set():
+ with st.lock:
+ if not todo:
+ return
+ i = todo.pop(0)
+ _repair_piece(src, st, f, i, stats)
+ st.finish(i)
+ except (_PieceFailed, _NoRange, _DiskError, _Changed, OSError) as e:
+ errors.append(e)
+ st.stop.set()
+
+ threads = [threading.Thread(target=worker, daemon=True) for _ in range(min(conns, st.n))]
+ for t in threads:
+ t.start()
+ last = 0.0
+ try:
+ while any(t.is_alive() for t in threads):
+ for t in threads:
+ t.join(0.25)
+ if time.time() - last > 2:
+ last = time.time()
+ have = st.have()
+ print(f"\r {label}: {have / 1e9:6.2f} / {total / 1e9:.2f} GB ({100 * have / total:.0f}%) ", end="", flush=True)
+ except BaseException:
+ st.stop.set()
+ raise
+ print()
+ if errors:
+ warn(f"could not compare {path.name} with the server again ({errors[0]})")
+ return None
+ try:
+ with open(path, "r+b") as g:
+ os.fsync(g.fileno())
+ except OSError:
+ pass
+ return len(stats["pieces"]), st.n
+
+
def download_single(url: str, part: Path, total: int, label: str):
- """One stream with a Range resume from the size of `part` (the way every download worked before parallel pieces)."""
+ """One stream with a Range resume from the size of `part` (the way every download worked before parallel pieces).
+ A resumed answer is used only when it is a 206 that starts at the byte asked for; anything else starts the file over."""
_single_ready(part)
- have = part.stat().st_size if part.exists() else 0
for attempt in range(30):
+ have = part.stat().st_size if part.exists() else 0
if total and have >= total: # stopped after the last byte, before the rename: nothing to
break # ask for (a range past the end is a 416, retried 30 times)
try:
req = urllib.request.Request(url, headers={"User-Agent": "strata-setup", "Range": f"bytes={have}-"})
- with _urlopen(req, timeout=60) as r, open(part, "ab" if have else "wb") as f:
- if have and r.status != 206: # the server ignored the range: start over
- f.seek(0)
- f.truncate()
- have = 0
- last = 0.0
- while True:
- b = r.read(8 << 20)
- if not b:
- break
- f.write(b)
- have += len(b)
- if time.time() - last > 2:
- last = time.time()
- size = f"{have / 1e9:6.2f} / {total / 1e9:.2f} GB ({100 * have / total:.0f}%)" if total \
- else f"{have / 1e6:7.1f} MB"
- print(f"\r {label}: {size} ", end="", flush=True)
+ with _urlopen(req, timeout=60) as r:
+ if have:
+ m = _CONTENT_RANGE.match(r.headers.get("Content-Range") or "")
+ if r.status != 206 or not m or int(m.group(1)) != have or (total and int(m.group(3)) != total):
+ if r.status != 200: # another range: its bytes are not the file's; ask again from the start
+ part.unlink(missing_ok=True)
+ raise OSError(f"the server answered another range than bytes {have}- (HTTP {r.status})")
+ have = 0 # the server ignored the range and sent the whole file: start over
+ elif r.status not in (200, 206):
+ raise OSError(f"HTTP {r.status}")
+ with open(part, "ab" if have else "wb") as f:
+ last = 0.0
+ while True:
+ b = r.read(8 << 20)
+ if not b:
+ break
+ if total and have + len(b) > total:
+ raise OSError("the server sent more bytes than the file has")
+ f.write(b)
+ have += len(b)
+ if time.time() - last > 2:
+ last = time.time()
+ size = f"{have / 1e9:6.2f} / {total / 1e9:.2f} GB ({100 * have / total:.0f}%)" if total else f"{have / 1e6:7.1f} MB"
+ print(f"\r {label}: {size} ", end="", flush=True)
+ f.flush()
+ os.fsync(f.fileno())
print()
if not total or have >= total:
break
except (OSError, http.client.HTTPException) as e:
print()
+ if part.exists() and total and part.stat().st_size > total:
+ part.unlink(missing_ok=True) # more than the file has can only be wrong bytes
warn(f"download interrupted ({e}); retrying in 10 s ...")
time.sleep(10)
@@ -2416,23 +2645,34 @@ def download(url, dst: Path, what=None):
if pin and total and total != pin[0]: # not the pinned file: stop before the download, not after
fail(f"{what or dst.name} is {total:,} bytes on {url.split('/')[2]}, not the pinned {pin[0]:,}",
"update Strata (the file there has changed since this version pinned it)")
- if dst.exists() and total and dst.stat().st_size == total: # finished by an older setup (no mark yet)
- mark(dst)
- ok(f"{what or dst.name} already downloaded")
- return
label = what or dst.name
conns = dl_connections()
+ meta = ms_meta(*ms) if ms is not None else None
+ fixed = pin if pin else (meta if meta is not None and meta[1] else None)
+ again = (lambda: repair_download(url, dst, total, label, conns)) if total >= DL_MIN_PARALLEL and conns > 1 else None
+ if dst.exists() and total and dst.stat().st_size == total: # finished by an older setup (no mark yet), or stopped before the check
+ if fixed: # a file that has a fixed hash is checked, not just marked
+ verify_sha256(dst, *fixed, repair=again)
+ else:
+ mark(dst)
+ ok(f"{what or dst.name} already downloaded")
+ return
if not (total >= DL_MIN_PARALLEL and conns > 1 and parallel_download(url, part, total, label, conns, dst.name)):
+ _claim_part(part, url)
download_single(url, part, total, label)
if total and part.stat().st_size != total:
fail(f"could not finish downloading {dst.name}: {part.stat().st_size:,} bytes on disk, the server says {total:,}",
"check your internet connection and run it again (the download resumes where it stopped)")
- part.replace(dst)
- meta = ms_meta(*ms) if ms is not None else None
+ try:
+ atomic.replace(part, dst) # waits a moment for a virus scanner or indexer that holds the file
+ except OSError as e:
+ fail(f"cannot finish {dst.name}: {e}", "close any program that has the file open and run setup again")
+ for side in (".chunks", ".src"):
+ part.with_name(part.name + side).unlink(missing_ok=True)
if pin:
- verify_sha256(dst, *pin) # the fixed hash: a wrong file is deleted and setup stops
+ verify_sha256(dst, *pin, repair=again) # the fixed hash: a wrong file is repaired, else deleted and setup stops
elif meta is not None and meta[1]:
- verify_sha256(dst, meta[0], meta[1]) # ModelScope's published hash; kept in the finish mark
+ verify_sha256(dst, meta[0], meta[1], repair=again) # ModelScope's published hash; kept in the finish mark
else:
if ms is not None:
warn(f"{what or dst.name}: ModelScope publishes no SHA-256 for it: the file is not verified")
@@ -2526,19 +2766,7 @@ def gguf_dir_problem(folder: Path, first: Path, fam: dict, model: str) -> tuple
return f"{folder} has no {fam['title']} {model} file", hint
-def verify_sha256(s: Path, size: int, sha: str, own: bool = False) -> None:
- """A shard's size and SHA-256 against the pinned values (the Unsloth file); the result is kept in its finish mark,
- so the ~5 minutes of hashing 111 GB happen once. A wrong file is deleted, so the next run downloads it again."""
- m = s.with_name(s.name + ".done")
- if m.exists() and f"sha256 {sha}" in m.read_text(encoding="utf-8", errors="replace"):
- return
- have = s.stat().st_size if s.exists() else -1
- if have != size:
- if own: # --gguf-dir: the person's own file is never deleted by setup
- fail(f"{s.name} is {have:,} bytes; the published file is {size:,}",
- "it may be cut short or another version; your file was left as it is. Copy the whole published file into the folder")
- fail(f"{s.name} is {have:,} bytes, not {size:,}", "delete it and run setup again (the download restarts)")
- say(f" checking {s.name} (SHA-256, {size / 1e9:.1f} GB) ...")
+def _hash_file(s: Path, size: int) -> str:
h = hashlib.sha256()
seen, last = 0, 0.0
with open(s, "rb") as f:
@@ -2553,13 +2781,40 @@ def verify_sha256(s: Path, size: int, sha: str, own: bool = False) -> None:
print(f"\r checking {s.name}: {seen / 1e9:6.2f} / {size / 1e9:.2f} GB ({100 * seen // size}%) ", end="", flush=True)
if seen >= (1 << 30):
print()
- if h.hexdigest() != sha:
+ return h.hexdigest()
+
+
+def verify_sha256(s: Path, size: int, sha: str, own: bool = False, repair=None) -> None:
+ """A shard's size and SHA-256 against the pinned values (the Unsloth file); the result is kept in its finish mark,
+ so the ~5 minutes of hashing 111 GB happen once. A wrong file is deleted, so the next run downloads it again.
+ `repair` (a downloaded file's own re-reader, see repair_download): the wrong file is first compared with the server
+ piece by piece and only the damaged pieces are written again, then it is checked a second time."""
+ m = s.with_name(s.name + ".done")
+ if m.exists() and f"sha256 {sha}" in m.read_text(encoding="utf-8", errors="replace"):
+ return
+ have = s.stat().st_size if s.exists() else -1
+ if have != size:
+ if own: # --gguf-dir: the person's own file is never deleted by setup
+ fail(f"{s.name} is {have:,} bytes; the published file is {size:,}",
+ "it may be cut short or another version; your file was left as it is. Copy the whole published file into the folder")
+ fail(f"{s.name} is {have:,} bytes, not {size:,}", "delete it and run setup again (the download restarts)")
+ say(f" checking {s.name} (SHA-256, {size / 1e9:.1f} GB) ...")
+ digest = _hash_file(s, size)
+ if digest != sha and repair is not None and not own:
+ say(f" Some parts of {s.name} were damaged; downloading only those parts again.")
+ res = repair()
+ if res is not None:
+ say(f" {res[0]} of {res[1]} parts were different and have been written again; checking the whole file again ...")
+ digest = _hash_file(s, size)
+ else:
+ say(" The damaged parts could not be found this way.")
+ if digest != sha:
if own:
- fail(f"{s.name} does not match the published file (SHA-256 {h.hexdigest()}, expected {sha})",
+ fail(f"{s.name} does not match the published file (SHA-256 {digest}, expected {sha})",
"it may be damaged or another version; your file was left as it is. Copy the whole published file into the folder")
s.unlink(missing_ok=True)
m.unlink(missing_ok=True)
- fail(f"{s.name} has the wrong SHA-256 ({h.hexdigest()}, expected {sha}): deleted",
+ fail(f"{s.name} has the wrong SHA-256 ({digest}, expected {sha}): deleted",
"run setup again to download it again")
mark(s, f"sha256 {sha}")
diff --git a/strata_control/transfer.py b/strata_control/transfer.py
index 256877319..e416b3777 100644
--- a/strata_control/transfer.py
+++ b/strata_control/transfer.py
@@ -15,6 +15,7 @@ import hashlib
import http.client
import ipaddress
import os
+import re
import threading
import email.utils
import datetime as _dt
@@ -228,8 +229,11 @@ def _transfer(location, partial: Path, have: int, size: int, progress, cancel, t
headers["Range"] = f"bytes={have}-"
request = urllib.request.Request(location, headers=headers)
with _opener.open(request, timeout=timeout) as response:
- if have and response.status != 206:
- have = done = 0 # the server ignored the range: start over, never append to a prefix
+ if have and not _resumes_at(response, have, size):
+ if response.status == 206: # another range: its bytes are not the file's, ask again from the start
+ partial.unlink(missing_ok=True)
+ return _transfer(location, partial, 0, size, progress, cancel, timeout)
+ have = done = 0 # the server ignored the range and sent the whole file: start over, never append to a prefix
_copy(response, partial, "wb" if not have else "ab", done, size, progress, cancel)
else:
path = _local_path(location)
@@ -252,6 +256,17 @@ def _transfer(location, partial: Path, have: int, size: int, progress, cancel, t
resumable=True) from None
+_CONTENT_RANGE = re.compile(r"\s*bytes (\d+)-(\d+)/(\d+|\*)\s*$")
+
+
+def _resumes_at(response, have: int, size: int) -> bool:
+ """Whether a resumed answer is a 206 whose Content-Range starts at byte `have` of a file of `size` bytes."""
+ if response.status != 206:
+ return False
+ m = _CONTENT_RANGE.match(response.headers.get("Content-Range") or "")
+ return bool(m) and int(m.group(1)) == have and (m.group(3) == "*" or int(m.group(3)) == size)
+
+
def _copy(source, partial: Path, mode: str, done: int, size: int, progress, cancel) -> None:
with partial.open(mode) as out:
while True:
diff --git a/tests/terminal/test_download_integrity.py b/tests/terminal/test_download_integrity.py
new file mode 100644
index 000000000..64b27654e
--- /dev/null
+++ b/tests/terminal/test_download_integrity.py
@@ -0,0 +1,443 @@
+"""0.2.2: a model download never leaves wrong bytes behind, and a file with a wrong SHA-256 is repaired, not thrown away.
+
+A user's 39.8 GB shard arrived with the wrong SHA-256 on 0.2.1 (16 connections, several pieces per request). These tests run
+setup.download() against a local HTTP server that misbehaves in each way a server, a CDN or a disk can: a 200 for a ranged
+request, a 206 for another range or another total, a body that ends early, another version of the file part-way, a changed
+ETag, a .part written from another mirror, no piece list, a full disk, a raw write that takes fewer bytes than given, and
+bytes that are wrong in the first download (the repair reads the file again and writes only the pieces that differ).
+Pieces are 64 KiB here instead of 32 MiB, so a few MiB stand for the 40 GB."""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import http.server
+import io
+import json
+import random
+import re
+import shutil
+import tempfile
+import threading
+import unittest
+from pathlib import Path
+from unittest import mock
+
+import setup
+from strata_control import transfer
+
+PIECE = 64 << 10
+SIZE = 40 * PIECE + 12345 # 41 pieces, the last one short
+DATA = random.Random(7).randbytes(SIZE)
+SHA = hashlib.sha256(DATA).hexdigest()
+
+
+class Behavior:
+ """What the server does with request number n (1 = the first). `plan` returns overrides for that answer."""
+
+ def __init__(self, plan=None, data=DATA, etag='"v1"'):
+ self.plan = plan or (lambda n, start, end: {})
+ self.data = data
+ self.etag = etag
+ self.lock = threading.Lock()
+ self.count = 0
+ self.served = [] # (start, end) of every ranged answer
+ self.once = set()
+
+ def next(self):
+ with self.lock:
+ self.count += 1
+ return self.count
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.0" # one request per connection: ending a body early is a reset
+
+ def log_message(self, *a):
+ pass
+
+ def do_HEAD(self):
+ beh = self.server.beh
+ self.send_response(200)
+ self.send_header("Content-Length", str(len(beh.data)))
+ self.send_header("Accept-Ranges", "bytes")
+ self.end_headers()
+
+ def do_GET(self):
+ beh = self.server.beh
+ n = beh.next()
+ total = len(beh.data)
+ m = re.match(r"bytes=(\d+)-(\d*)", self.headers.get("Range") or "")
+ start = int(m.group(1)) if m else 0
+ end = min(int(m.group(2)), total - 1) if m and m.group(2) else total - 1
+ if start >= total:
+ self.send_response(416)
+ self.end_headers()
+ return
+ o = beh.plan(n, start, end) or {}
+ if o.get("status") == 404 or o.get("status") == 503:
+ self.send_response(o["status"])
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+ return
+ status = o.get("status", 206 if m else 200)
+ b0, b1 = o.get("body", (start, end))
+ body = bytearray(beh.data if status == 200 else beh.data[b0:b1 + 1])
+ for off in o.get("corrupt", ()):
+ lo = 0 if status == 200 else b0
+ if lo <= off < lo + len(body):
+ body[off - lo] ^= 0xFF
+ self.send_response(status)
+ self.send_header("Content-Length", str(o.get("cl", len(body))))
+ if status == 206:
+ self.send_header("Content-Range", o.get("cr", f"bytes {start}-{end}/{total}"))
+ self.send_header("ETag", o.get("etag", beh.etag))
+ self.end_headers()
+ with beh.lock:
+ beh.served.append((start, end))
+ cut = o.get("cut")
+ try:
+ self.wfile.write(bytes(body[:cut] if cut is not None else body))
+ except OSError:
+ pass
+
+
+class Server(http.server.ThreadingHTTPServer):
+ daemon_threads = True
+
+ def handle_error(self, request, client_address):
+ pass
+
+
+class DownloadCase(unittest.TestCase):
+ def setUp(self):
+ self.tmp = Path(tempfile.mkdtemp(prefix="dlint"))
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+ patches = [mock.patch.object(setup, "DL_CHUNK", PIECE), mock.patch.object(setup, "DL_MIN_PARALLEL", PIECE * 4),
+ mock.patch.object(setup, "DL_BACKOFF", 0.0), mock.patch.object(setup, "DL_SAVE_EVERY", 0.0),
+ mock.patch.object(setup, "DL_ROUNDS", 3), mock.patch.object(setup.time, "sleep", lambda s: None),
+ mock.patch.object(setup, "file_pin", lambda url: (SIZE, SHA)),
+ mock.patch.dict("os.environ", {"STRATA_DL_CONNECTIONS": "4", "STRATA_SOURCE": "huggingface"})]
+ for p in patches:
+ p.start()
+ self.addCleanup(p.stop)
+ self.dst = self.tmp / "model.gguf"
+ self.part = self.tmp / "model.gguf.part"
+
+ def serve(self, beh):
+ srv = Server(("127.0.0.1", 0), Handler)
+ srv.beh = beh
+ threading.Thread(target=srv.serve_forever, daemon=True).start()
+ self.addCleanup(srv.server_close)
+ self.addCleanup(srv.shutdown)
+ self.beh = beh
+ return f"http://127.0.0.1:{srv.server_address[1]}/m/model.gguf"
+
+ def run_download(self, url, expect_exit=False):
+ buf = io.StringIO()
+ with contextlib.redirect_stdout(buf):
+ if expect_exit:
+ with self.assertRaises(SystemExit):
+ setup.download(url, self.dst)
+ else:
+ setup.download(url, self.dst)
+ self.out = buf.getvalue()
+ return self.out
+
+ def assertGood(self):
+ self.assertTrue(self.dst.is_file(), self.out)
+ self.assertEqual(hashlib.sha256(self.dst.read_bytes()).hexdigest(), SHA)
+ self.assertFalse(self.part.exists())
+ self.assertFalse(self.tmp.joinpath("model.gguf.part.chunks").exists())
+
+
+class Misbehaving(DownloadCase):
+ def test_clean_download(self):
+ self.run_download(self.serve(Behavior()))
+ self.assertGood()
+
+ def test_ranged_requests_answered_with_200_use_one_stream(self):
+ self.run_download(self.serve(Behavior(lambda n, s, e: {"status": 200})))
+ self.assertGood()
+
+ def test_ranges_stop_being_honoured_part_way(self):
+ # the probe and the first pieces are right; later requests get the whole file with a 200
+ self.run_download(self.serve(Behavior(lambda n, s, e: {"status": 200} if n > 6 else {})))
+ self.assertGood()
+
+ def test_206_for_another_range_is_not_written(self):
+ # Content-Range names a range that starts later than the one asked for, from the 5th request on
+ self.run_download(self.serve(Behavior(lambda n, s, e: {"cr": f"bytes {s + 7}-{e}/{SIZE}", "body": (s + 7, e)}
+ if n > 4 and s > 0 else {})))
+ self.assertGood()
+
+ def test_206_with_another_total_is_not_written(self):
+ self.run_download(self.serve(Behavior(lambda n, s, e: {"cr": f"bytes {s}-{e}/{SIZE + 1}"} if n > 4 else {})))
+ self.assertGood()
+
+ def test_206_with_a_longer_body_than_asked(self):
+ # Content-Length does not fit the range: tried again, then right
+ self.run_download(self.serve(Behavior(lambda n, s, e: {"cl": e - s + 5} if n in (4, 5, 9) else {})))
+ self.assertGood()
+
+ def test_connection_reset_in_the_middle_of_pieces(self):
+ # every 3rd answer ends after a random part of its body; the piece it broke in is written again from its start
+ rnd = random.Random(3)
+ self.run_download(self.serve(Behavior(lambda n, s, e: {"cut": rnd.randrange(1, max(2, e - s))} if n % 3 == 0 else {})))
+ self.assertGood()
+
+ def test_an_answer_cut_after_a_whole_number_of_blocks_still_ends_in_the_right_bytes(self):
+ self.run_download(self.serve(Behavior(lambda n, s, e: {"cut": PIECE + 17} if n in (3, 4, 5, 6) else {})))
+ self.assertGood()
+
+ def test_one_answer_from_another_version_is_asked_again(self):
+ self.run_download(self.serve(Behavior(lambda n, s, e: {"etag": '"v2"'} if n in (4, 7) else {})))
+ self.assertGood()
+
+ def test_another_version_for_good_stops_and_leaves_no_file(self):
+ out = self.run_download(self.serve(Behavior(lambda n, s, e: {"etag": '"v2"'} if n > 3 else {})), expect_exit=True)
+ self.assertFalse(self.dst.exists())
+
+ def test_the_file_changing_between_resolves_is_noticed(self):
+ src = setup._Source("http://x/y", SIZE)
+ src.etag = "aaa"
+
+ class R:
+ status = 206
+ headers = {"Content-Range": f"bytes 0-0/{SIZE}", "ETag": '"bbb"'}
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *a):
+ return False
+
+ def read(self, n):
+ return b"x"
+
+ def geturl(self):
+ return "http://x/y"
+
+ with mock.patch.object(setup, "_urlopen", lambda *a, **k: R()):
+ with self.assertRaises(setup._Changed):
+ src.resolve()
+
+ def test_server_error_part_way_keeps_pieces_and_a_second_run_finishes(self):
+ url = self.serve(Behavior(lambda n, s, e: {"status": 503} if n > 12 else {}))
+ self.run_download(url, expect_exit=True)
+ self.assertTrue(self.part.exists())
+ self.assertTrue(self.tmp.joinpath("model.gguf.part.chunks").exists())
+ first_run = len(self.beh.served)
+ self.beh.plan = lambda n, s, e: {} # the same address, now answering
+ self.beh.served.clear()
+ self.run_download(url)
+ self.assertGood()
+ fetched = sum(e - s + 1 for s, e in self.beh.served if e > s) # without the one-byte probes
+ self.assertLess(fetched, SIZE, "the second run took the whole file again instead of resuming")
+ self.assertGreater(first_run, 1)
+
+ def test_two_interrupted_runs_then_a_good_one(self):
+ url = self.serve(Behavior())
+ for limit in (6, 6):
+ self.beh.count = 0
+ self.beh.plan = lambda n, s, e, limit=limit: {"status": 503} if n > limit else {}
+ self.run_download(url, expect_exit=True)
+ self.beh.plan = lambda n, s, e: {}
+ self.run_download(url)
+ self.assertGood()
+
+ def test_a_part_written_from_another_mirror_is_dropped(self):
+ url = self.serve(Behavior())
+ self.part.write_bytes(b"\xaa" * (PIECE * 10))
+ self.tmp.joinpath("model.gguf.part.chunks").write_text(
+ json.dumps({"v": 1, "total": SIZE, "chunk": PIECE, "done": [[0, 10]]}), encoding="utf-8")
+ self.tmp.joinpath("model.gguf.part.src").write_text(
+ json.dumps({"url": "https://www.modelscope.cn/models/x/resolve/master/model.gguf", "etag": None}), encoding="utf-8")
+ self.run_download(url)
+ self.assertGood() # the 0xaa pieces were not trusted
+
+ def test_a_part_of_another_version_is_dropped(self):
+ url = self.serve(Behavior())
+ self.part.write_bytes(b"\xaa" * (PIECE * 10))
+ self.tmp.joinpath("model.gguf.part.chunks").write_text(
+ json.dumps({"v": 1, "total": SIZE, "chunk": PIECE, "done": [[0, 10]]}), encoding="utf-8")
+ self.tmp.joinpath("model.gguf.part.src").write_text(json.dumps({"url": url, "etag": '"old"'}), encoding="utf-8")
+ self.run_download(url)
+ self.assertGood()
+
+ def test_without_a_writable_piece_list_the_file_is_not_written_in_parts(self):
+ url = self.serve(Behavior())
+ with mock.patch.object(setup, "_save_pieces", return_value=False):
+ self.run_download(url)
+ self.assertGood()
+
+ def test_a_stale_piece_list_never_names_bytes_the_part_lacks(self):
+ self.part.write_bytes(b"\x00" * (PIECE * 3))
+ self.tmp.joinpath("model.gguf.part.chunks").write_text(
+ json.dumps({"v": 1, "total": SIZE, "chunk": PIECE, "done": [[0, 30]]}), encoding="utf-8")
+ self.assertEqual(setup._load_pieces(self.part, SIZE), {0, 1, 2})
+
+ def test_full_disk_stops_with_a_plain_message_and_keeps_the_part(self):
+ url = self.serve(Behavior())
+
+ def full(f, pos, data):
+ raise OSError(28, "No space left on device")
+
+ with mock.patch.object(setup, "_write_all", full):
+ out = self.run_download(url, expect_exit=True)
+ self.assertIn("cannot write", out)
+ self.assertIn("free some disk space", out)
+ self.assertFalse(self.dst.exists())
+
+ def test_a_raw_write_that_takes_fewer_bytes_is_finished(self):
+ class Slow:
+ def __init__(self):
+ self.buf = bytearray(100)
+ self.pos = 0
+
+ def seek(self, p):
+ self.pos = p
+
+ def write(self, v):
+ n = min(7, len(v))
+ self.buf[self.pos:self.pos + n] = bytes(v[:n])
+ self.pos += n
+ return n
+
+ f = Slow()
+ setup._write_all(f, 10, bytes(range(60)))
+ self.assertEqual(bytes(f.buf[10:70]), bytes(range(60)))
+
+ def test_a_raw_write_that_takes_nothing_is_an_error(self):
+ class Zero:
+ def seek(self, p):
+ pass
+
+ def write(self, v):
+ return 0
+
+ with self.assertRaises(OSError):
+ setup._write_all(Zero(), 0, b"abc")
+
+ def test_single_stream_resume_ignores_a_206_from_the_wrong_offset(self):
+ url = self.serve(Behavior(lambda n, s, e: {"cr": f"bytes {s + 5}-{e}/{SIZE}", "body": (s + 5, e)} if s > 0 else {}))
+ self.part.write_bytes(DATA[:PIECE * 3])
+ with mock.patch.dict("os.environ", {"STRATA_DL_CONNECTIONS": "1"}):
+ self.run_download(url)
+ self.assertGood()
+
+ def test_single_stream_resume_ignores_a_200(self):
+ url = self.serve(Behavior(lambda n, s, e: {"status": 200}))
+ self.part.write_bytes(DATA[:PIECE * 3])
+ with mock.patch.dict("os.environ", {"STRATA_DL_CONNECTIONS": "1"}):
+ self.run_download(url)
+ self.assertGood()
+
+ def test_finished_file_without_a_mark_is_checked_not_marked(self):
+ self.dst.write_bytes(DATA[:-1] + b"\x00")
+ url = self.serve(Behavior())
+ self.run_download(url)
+ self.assertIn("Some parts of model.gguf were damaged", self.out)
+ self.assertGood()
+ self.assertTrue(self.tmp.joinpath("model.gguf.done").exists())
+
+
+class Repair(DownloadCase):
+ def test_wrong_bytes_in_two_pieces_are_found_and_only_those_rewritten(self):
+ bad = [5 * PIECE + 100, 30 * PIECE + 7]
+ seen = set()
+
+ def plan(n, s, e):
+ hit = [o for o in bad if s <= o <= e and o not in seen]
+ seen.update(hit)
+ return {"corrupt": hit} if hit else {}
+
+ out = self.run_download(self.serve(Behavior(plan)))
+ self.assertGood()
+ self.assertIn("Some parts of model.gguf were damaged; downloading only those parts again", out)
+ self.assertIn("2 of 41 parts were different", out)
+ self.assertTrue(self.tmp.joinpath("model.gguf.done").exists())
+
+ def test_one_flipped_byte_in_the_last_short_piece(self):
+ bad = [SIZE - 3]
+ seen = set()
+
+ def plan(n, s, e):
+ hit = [o for o in bad if s <= o <= e and o not in seen]
+ seen.update(hit)
+ return {"corrupt": hit} if hit else {}
+
+ out = self.run_download(self.serve(Behavior(plan)))
+ self.assertGood()
+ self.assertIn("1 of 41 parts were different", out)
+
+ def test_a_repair_that_loses_the_connection_part_way_still_ends_right(self):
+ bad = [3 * PIECE + 1]
+ seen = set()
+
+ def plan(n, s, e):
+ hit = [o for o in bad if s <= o <= e and o not in seen]
+ seen.update(hit)
+ o = {"corrupt": hit} if hit else {}
+ if seen and n % 4 == 0:
+ o["cut"] = 5000
+ return o
+
+ self.run_download(self.serve(Behavior(plan)))
+ self.assertGood()
+
+ def test_a_server_that_keeps_serving_the_wrong_file_ends_with_the_file_deleted(self):
+ wrong = bytearray(DATA)
+ wrong[1000] ^= 0xFF
+ url = self.serve(Behavior(data=bytes(wrong)))
+ out = self.run_download(url, expect_exit=True)
+ self.assertIn("wrong SHA-256", out)
+ self.assertFalse(self.dst.exists())
+ self.assertFalse(self.tmp.joinpath("model.gguf.done").exists())
+
+ def test_the_repair_is_off_for_a_file_the_person_owns(self):
+ self.dst.write_bytes(b"x" * SIZE)
+ called = []
+ with contextlib.redirect_stdout(io.StringIO()):
+ with self.assertRaises(SystemExit):
+ setup.verify_sha256(self.dst, SIZE, SHA, own=True, repair=lambda: called.append(1))
+ self.assertEqual(called, [])
+ self.assertTrue(self.dst.exists())
+
+
+class UpdateTransfer(unittest.TestCase):
+ """strata_control/transfer.py (update archives): a resumed answer is appended only when it starts at the byte asked for."""
+
+ def setUp(self):
+ self.tmp = Path(tempfile.mkdtemp(prefix="dlint"))
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+
+ def test_resume_check(self):
+ class R:
+ def __init__(self, status, cr):
+ self.status = status
+ self.headers = {"Content-Range": cr} if cr else {}
+
+ self.assertTrue(transfer._resumes_at(R(206, "bytes 100-999/1000"), 100, 1000))
+ self.assertFalse(transfer._resumes_at(R(206, "bytes 0-999/1000"), 100, 1000))
+ self.assertFalse(transfer._resumes_at(R(206, "bytes 100-999/2000"), 100, 1000))
+ self.assertFalse(transfer._resumes_at(R(206, ""), 100, 1000))
+ self.assertFalse(transfer._resumes_at(R(200, "bytes 100-999/1000"), 100, 1000))
+
+ def test_a_206_from_the_wrong_offset_restarts_the_file(self):
+ data = DATA[:200000]
+ beh = Behavior(lambda n, s, e: {"cr": f"bytes {s + 9}-{e}/{len(data)}", "body": (s + 9, e)} if s > 0 else {}, data=data)
+ srv = Server(("127.0.0.1", 0), Handler)
+ srv.beh = beh
+ threading.Thread(target=srv.serve_forever, daemon=True).start()
+ self.addCleanup(srv.server_close)
+ self.addCleanup(srv.shutdown)
+ part = self.tmp / "a.part"
+ part.write_bytes(data[:50000])
+ got = transfer.download(f"http://127.0.0.1:{srv.server_address[1]}/a", part, size=len(data),
+ sha256=hashlib.sha256(data).hexdigest())
+ self.assertEqual(got.read_bytes(), data)
+
+
+if __name__ == "__main__":
+ unittest.main()