Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Downloads: every ranged answer is checked, a file with a wrong SHA-256 is repaired piece by piece instead of deleted
Niko122163214a565580
diff --git a/setup.py b/setup.pyindex 8d9f89c5a..723d84141 100644--- a/setup.py+++ b/setup.py@@ -63,6 +63,7 @@ import zipfilefrom pathlib import Pathfrom strata_control import acquisition+from strata_control import atomicfrom strata_control import draft_layerfrom strata_control.discovery import in_trashfrom strata_control import user_args@@ -1975,6 +1976,63 @@ class _PieceFailed(Exception):"""A piece failed DL_CHUNK_TRIES times."""+class _Changed(Exception):+ """The server's file is not the one the first pieces came from (another ETag): what was written cannot be trusted."""+++_CONTENT_RANGE = re.compile(r"\s*bytes (\d+)-(\d+)/(\d+)\s*$")+++def _etag(headers) -> str | None:+ """The object identity a response names: ETag without the weak mark and quotes; None when it sends none."""+ raw = (headers.get("ETag") or "").strip()+ if raw[:2] in ("W/", "w/"):+ raw = raw[2:]+ return raw.strip('"') or None+++def _check_ranged(r, pos: int, end: int, total: int, etag: str | None):+ """A ranged answer is trusted only when it is exactly the bytes asked for: status 206, Content-Range `pos-end/total`+ (a 200 with the whole file, another range or another total is the server not honouring the request), a Content-Length+ that fits, and the same object (ETag) as the first answer. _NoRange: this server cannot be used with ranges;+ OSError: a bad answer worth asking again (another edge)."""+ if r.status != 206:+ raise _NoRange(f"HTTP {r.status}")+ m = _CONTENT_RANGE.match(r.headers.get("Content-Range") or "")+ if not m or (int(m.group(1)), int(m.group(2)), int(m.group(3))) != (pos, end, total):+ raise _NoRange(f"Content-Range {(r.headers.get('Content-Range') or '')!r} for bytes {pos}-{end}/{total}")+ cl = (r.headers.get("Content-Length") or "").strip()+ if cl and cl.isdigit() and int(cl) != end - pos + 1:+ raise OSError(f"Content-Length {cl} for {end - pos + 1} bytes asked")+ got = _etag(r.headers)+ if etag and got and got != etag:+ raise OSError("the server answered with another version of the file")+++def _write_all(f, pos: int, data):+ """Write `data` at `pos`; a raw unbuffered file may take fewer bytes than it was given, so go on until all are down."""+ view = memoryview(data)+ f.seek(pos)+ done_n = 0+ while done_n < len(view):+ n = f.write(view[done_n:])+ if not n:+ raise OSError("the disk took no more bytes")+ done_n += n+++def _read_exact(f, pos: int, n: int) -> bytes:+ f.seek(pos)+ parts = []+ while n > 0:+ b = f.read(n)+ if not b:+ break+ parts.append(b)+ n -= len(b)+ return b"".join(parts)++def _url_ttl(url: str):"""Seconds a signed URL (the CDN's, after the redirect) is still valid by its own query, else None."""q = urllib.parse.parse_qs(urllib.parse.urlsplit(url).query)@@ -1998,6 +2056,7 @@ class _Source:def __init__(self, url: str, total: int):self.url, self.total = url, totalself.final, self.deadline = None, None+ self.etag = None # the object the first answer named; every later answer must matchself.lock = threading.Lock()def resolve(self, stale=None) -> str:@@ -2007,8 +2066,12 @@ class _Source:req = urllib.request.Request(self.url, headers={"User-Agent": "strata-setup", "Range": "bytes=0-0"})with _urlopen(req, timeout=60) as r:cr = (r.headers.get("Content-Range") or "").strip()- if r.status != 206 or cr not in (f"bytes 0-0/{self.total}", "bytes 0-0/*"):+ if r.status != 206 or cr != f"bytes 0-0/{self.total}":raise _NoRange(f"HTTP {r.status}, Content-Range {cr!r}")+ tag = _etag(r.headers)+ if self.etag and tag and tag != self.etag:+ raise _Changed(f"the file on the server changed while it was downloaded ({self.etag[:12]} -> {tag[:12]})")+ self.etag = self.etag or tagr.read(1)final = r.geturl() or self.urlttl = _url_ttl(final)@@ -2100,17 +2163,45 @@ def _load_pieces(part: Path, total: int) -> set:return set(range(n)) if size >= total else set(range(size // DL_CHUNK))-def _save_pieces(part: Path, total: int, done: set):- """Flush the data first, then record the pieces: a list never names bytes the disk does not have."""+def _save_pieces(part: Path, total: int, done: set) -> bool:+ """Flush the data first, then record the pieces: a list never names bytes the disk does not have. False when the+ list could not be written (an older list stays, which only costs re-downloading a few pieces after a stop)."""try:with open(part, "r+b") as g:os.fsync(g.fileno())side = part.with_name(part.name + ".chunks")tmp = side.with_name(side.name + ".tmp")tmp.write_text(json.dumps({"v": 1, "total": total, "chunk": DL_CHUNK, "done": _ranges(done)}), encoding="utf-8")- os.replace(tmp, side)+ with open(tmp, "r+b") as g:+ os.fsync(g.fileno())+ atomic.replace(tmp, side)+ return Trueexcept OSError:- pass # only costs re-downloading a few pieces after a stop+ return False+++def _claim_part(part: Path, url: str, etag: str | None = None):+ """Record where a .part's bytes come from (<name>.part.src: the address asked, and the file's ETag when the server+ names one). A .part written from another address (the other mirror, another revision) or from another version of+ the file is dropped with its piece list: two sources are not mixed inside one file, even when they should hold the+ same bytes. A .part from an older setup has no record and is kept as before."""+ src = part.with_name(part.name + ".src")+ try:+ old = json.loads(src.read_text(encoding="utf-8")) if src.is_file() else None+ except (OSError, ValueError):+ old = {}+ if old is not None:+ same = isinstance(old, dict) and old.get("url") == url and not (old.get("etag") and etag and old.get("etag") != etag)+ if not same:+ part.unlink(missing_ok=True)+ part.with_name(part.name + ".chunks").unlink(missing_ok=True)+ src.unlink(missing_ok=True)+ old = None+ if old is None or (etag and not old.get("etag")):+ try:+ src.write_text(json.dumps({"url": url, "etag": etag}), encoding="utf-8")+ except OSError:+ passdef _single_ready(part: Path):@@ -2150,7 +2241,8 @@ def _take_run(st: _Pieces, todo: list) -> list:def _fetch_run(src: _Source, st: _Pieces, f, run: list):"""One ranged request for the neighbouring pieces in `run`, written at their offsets; each piece counts as finished- when its last byte is down. A connection that breaks continues from the byte it reached, not from the piece start."""+ when its last byte is down. Every answer is checked to be exactly the range asked for (status, Content-Range, length,+ ETag); a connection that breaks continues from the byte it reached."""start = run[0] * DL_CHUNKend = min((run[-1] + 1) * DL_CHUNK, st.total) - 1pos, tries = start, 0@@ -2160,13 +2252,13 @@ def _fetch_run(src: _Source, st: _Pieces, f, run: list):while True:if st.stop.is_set():return- url, err, got, wait, ra = None, None, False, None, 0.0+ url, err, wait, ra = None, None, None, 0.0+ attempt_pos = postry:url = src.get()req = urllib.request.Request(url, headers={"User-Agent": "strata-setup", "Range": f"bytes={pos}-{end}"})with _urlopen(req, timeout=60) as r:- if r.status != 206 or not (r.headers.get("Content-Range") or "").strip().startswith(f"bytes {pos}-{end}/"):- raise _NoRange(f"HTTP {r.status}")+ _check_ranged(r, pos, end, st.total, src.etag)while pos <= end:if st.stop.is_set():return@@ -2176,12 +2268,10 @@ def _fetch_run(src: _Source, st: _Pieces, f, run: list):if not b:raise IOError("the connection closed early")try:- f.seek(pos)- f.write(b)+ _write_all(f, pos, b)except OSError as e:raise _DiskError(str(e)) from epos += len(b)- got = Truest.add(cur, len(b))if pos == cur_end:st.finish(cur)@@ -2192,7 +2282,7 @@ def _fetch_run(src: _Source, st: _Pieces, f, run: list):st.flight.pop(i, None)returnreturn- except (_NoRange, _DiskError):+ except (_NoRange, _DiskError, _Changed):raiseexcept urllib.error.HTTPError as e:err = e@@ -2200,7 +2290,7 @@ def _fetch_run(src: _Source, st: _Pieces, f, run: list):try:src.resolve(stale=url)wait = 0- except _NoRange:+ except (_NoRange, _Changed):raiseexcept (OSError, http.client.HTTPException):pass@@ -2211,7 +2301,11 @@ def _fetch_run(src: _Source, st: _Pieces, f, run: list):passexcept (OSError, http.client.HTTPException) as e:err = e- tries = 1 if got else tries + 1 # a try that moved forward is not a failed one+ # A try that moved forward is not a failed one. The request goes on from the byte it reached: what came before the+ # break is exactly the range that was asked for (the answer's status, Content-Range, length and ETag were checked and+ # every byte was written in full), and writing a piece again from its start would make a route that resets more often+ # than every DL_CHUNK bytes unable to finish any piece. The whole-file SHA-256 and the repair are the net behind it.+ tries = 1 if pos > attempt_pos else tries + 1if tries >= DL_CHUNK_TRIES:raise _PieceFailed(str(err))st.stop.wait(max(DL_BACKOFF * 2 ** (tries - 1), ra) if wait is None else wait)@@ -2228,7 +2322,7 @@ def _piece_worker(src: _Source, st: _Pieces, todo: list, part: Path):except _PieceFailed as e:st.errors.append(e)st.abort_round.set()- except (_NoRange, _DiskError) as e:+ except (_NoRange, _DiskError, _Changed) as e:st.fatal = st.fatal or est.stop.set()except OSError as e: # the .part itself could not be opened@@ -2246,11 +2340,14 @@ def parallel_download(url: str, part: Path, total: int, label: str, conns: int,return Falseexcept (OSError, http.client.HTTPException):return False # the one-stream path has its own retries and messages+ part.parent.mkdir(parents=True, exist_ok=True)+ _claim_part(part, url, src.etag) # a .part from another address or version is dropped heredone = _load_pieces(part, total)st = _Pieces(total, done, conns)- part.parent.mkdir(parents=True, exist_ok=True)if not part.exists():part.touch()+ if not part.with_name(part.name + ".chunks").is_file() and not _save_pieces(part, total, done):+ return False # no piece list can be kept here: the pieces would not be told from holeslast_print = last_save = 0.0def report(force=False):@@ -2292,6 +2389,12 @@ def parallel_download(url: str, part: Path, total: int, label: str, conns: int,raisereport(force=True)print()+ if isinstance(st.fatal, _Changed):+ part.unlink(missing_ok=True)+ part.with_name(part.name + ".chunks").unlink(missing_ok=True)+ part.with_name(part.name + ".src").unlink(missing_ok=True)+ fail(f"{name} changed on the server while it was downloading: the part that was there was deleted",+ "run it again (the download starts over with the file that is there now)")if isinstance(st.fatal, _DiskError):_save_pieces(part, total, st.snapshot())fail(f"cannot write {part.name}: {st.fatal}", "free some disk space (or pick another --models-dir) and run it again")@@ -2313,37 +2416,163 @@ def parallel_download(url: str, part: Path, total: int, label: str, conns: int,return True+def _repair_piece(src: _Source, st: _Pieces, f, i: int, stats: dict):+ """Read piece `i` from the server in 1 MiB blocks and compare each with what is on the disk; a block that differs is+ written again. A broken connection asks again from the block it reached (every block is final once compared)."""+ pos, end = i * DL_CHUNK, min((i + 1) * DL_CHUNK, st.total) - 1+ tries = 0+ while pos <= end:+ if st.stop.is_set():+ return+ url, wait, ra = None, None, 0.0+ before = pos+ try:+ url = src.get()+ req = urllib.request.Request(url, headers={"User-Agent": "strata-setup", "Range": f"bytes={pos}-{end}"})+ with _urlopen(req, timeout=60) as r:+ _check_ranged(r, pos, end, st.total, src.etag)+ while pos <= end:+ if st.stop.is_set():+ return+ b = r.read(min(1 << 20, end + 1 - pos))+ if not b:+ raise IOError("the connection closed early")+ try:+ if _read_exact(f, pos, len(b)) != b:+ _write_all(f, pos, b)+ with st.lock:+ stats["bytes"] += len(b)+ stats["pieces"].add(i)+ except OSError as e:+ raise _DiskError(str(e)) from e+ pos += len(b)+ st.add(i, len(b))+ return+ except (_NoRange, _DiskError, _Changed):+ raise+ except urllib.error.HTTPError as e:+ if e.code in (401, 403, 410) and url is not None:+ try:+ src.resolve(stale=url)+ wait = 0+ except (_NoRange, _Changed):+ raise+ except (OSError, http.client.HTTPException):+ pass+ elif e.code in (429, 503):+ try:+ ra = min(float(e.headers.get("Retry-After") or 0), 60)+ except (TypeError, ValueError):+ pass+ err = e+ except (OSError, http.client.HTTPException) as e:+ err = e+ tries = 1 if pos > before else tries + 1+ if tries >= DL_CHUNK_TRIES:+ raise _PieceFailed(str(err))+ st.stop.wait(max(DL_BACKOFF * 2 ** (tries - 1), ra) if wait is None else wait)+++def repair_download(url: str, path: Path, total: int, label: str, conns: int):+ """A finished file whose SHA-256 is wrong is not thrown away: read it again from the server piece by piece, compare+ each block with the disk and write only the blocks that differ. Returns (damaged pieces, pieces), or None when the+ server cannot do it (no ranges, or the file there is another one); the caller checks the SHA-256 again."""+ if not total or path.stat().st_size != total:+ return None+ src = _Source(url, total)+ try:+ src.resolve()+ except (_NoRange, _Changed, OSError, http.client.HTTPException):+ return None+ st = _Pieces(total, set(), conns)+ stats = {"bytes": 0, "pieces": set()}+ todo = list(range(st.n))+ errors = []++ def worker():+ try:+ with open(path, "r+b", buffering=0) as f:+ while not st.stop.is_set():+ with st.lock:+ if not todo:+ return+ i = todo.pop(0)+ _repair_piece(src, st, f, i, stats)+ st.finish(i)+ except (_PieceFailed, _NoRange, _DiskError, _Changed, OSError) as e:+ errors.append(e)+ st.stop.set()++ threads = [threading.Thread(target=worker, daemon=True) for _ in range(min(conns, st.n))]+ for t in threads:+ t.start()+ last = 0.0+ try:+ while any(t.is_alive() for t in threads):+ for t in threads:+ t.join(0.25)+ if time.time() - last > 2:+ last = time.time()+ have = st.have()+ print(f"\r {label}: {have / 1e9:6.2f} / {total / 1e9:.2f} GB ({100 * have / total:.0f}%) ", end="", flush=True)+ except BaseException:+ st.stop.set()+ raise+ print()+ if errors:+ warn(f"could not compare {path.name} with the server again ({errors[0]})")+ return None+ try:+ with open(path, "r+b") as g:+ os.fsync(g.fileno())+ except OSError:+ pass+ return len(stats["pieces"]), st.n++def download_single(url: str, part: Path, total: int, label: str):- """One stream with a Range resume from the size of `part` (the way every download worked before parallel pieces)."""+ """One stream with a Range resume from the size of `part` (the way every download worked before parallel pieces).+ A resumed answer is used only when it is a 206 that starts at the byte asked for; anything else starts the file over."""_single_ready(part)- have = part.stat().st_size if part.exists() else 0for attempt in range(30):+ have = part.stat().st_size if part.exists() else 0if total and have >= total: # stopped after the last byte, before the rename: nothing tobreak # ask for (a range past the end is a 416, retried 30 times)try:req = urllib.request.Request(url, headers={"User-Agent": "strata-setup", "Range": f"bytes={have}-"})- with _urlopen(req, timeout=60) as r, open(part, "ab" if have else "wb") as f:- if have and r.status != 206: # the server ignored the range: start over- f.seek(0)- f.truncate()- have = 0- last = 0.0- while True:- b = r.read(8 << 20)- if not b:- break- f.write(b)- have += len(b)- if time.time() - last > 2:- last = time.time()- size = f"{have / 1e9:6.2f} / {total / 1e9:.2f} GB ({100 * have / total:.0f}%)" if total \- else f"{have / 1e6:7.1f} MB"- print(f"\r {label}: {size} ", end="", flush=True)+ with _urlopen(req, timeout=60) as r:+ if have:+ m = _CONTENT_RANGE.match(r.headers.get("Content-Range") or "")+ if r.status != 206 or not m or int(m.group(1)) != have or (total and int(m.group(3)) != total):+ if r.status != 200: # another range: its bytes are not the file's; ask again from the start+ part.unlink(missing_ok=True)+ raise OSError(f"the server answered another range than bytes {have}- (HTTP {r.status})")+ have = 0 # the server ignored the range and sent the whole file: start over+ elif r.status not in (200, 206):+ raise OSError(f"HTTP {r.status}")+ with open(part, "ab" if have else "wb") as f:+ last = 0.0+ while True:+ b = r.read(8 << 20)+ if not b:+ break+ if total and have + len(b) > total:+ raise OSError("the server sent more bytes than the file has")+ f.write(b)+ have += len(b)+ if time.time() - last > 2:+ last = time.time()+ size = f"{have / 1e9:6.2f} / {total / 1e9:.2f} GB ({100 * have / total:.0f}%)" if total else f"{have / 1e6:7.1f} MB"+ print(f"\r {label}: {size} ", end="", flush=True)+ f.flush()+ os.fsync(f.fileno())print()if not total or have >= total:breakexcept (OSError, http.client.HTTPException) as e:print()+ if part.exists() and total and part.stat().st_size > total:+ part.unlink(missing_ok=True) # more than the file has can only be wrong byteswarn(f"download interrupted ({e}); retrying in 10 s ...")time.sleep(10)@@ -2416,23 +2645,34 @@ def download(url, dst: Path, what=None):if pin and total and total != pin[0]: # not the pinned file: stop before the download, not afterfail(f"{what or dst.name} is {total:,} bytes on {url.split('/')[2]}, not the pinned {pin[0]:,}","update Strata (the file there has changed since this version pinned it)")- if dst.exists() and total and dst.stat().st_size == total: # finished by an older setup (no mark yet)- mark(dst)- ok(f"{what or dst.name} already downloaded")- returnlabel = what or dst.nameconns = dl_connections()+ meta = ms_meta(*ms) if ms is not None else None+ fixed = pin if pin else (meta if meta is not None and meta[1] else None)+ again = (lambda: repair_download(url, dst, total, label, conns)) if total >= DL_MIN_PARALLEL and conns > 1 else None+ if dst.exists() and total and dst.stat().st_size == total: # finished by an older setup (no mark yet), or stopped before the check+ if fixed: # a file that has a fixed hash is checked, not just marked+ verify_sha256(dst, *fixed, repair=again)+ else:+ mark(dst)+ ok(f"{what or dst.name} already downloaded")+ returnif not (total >= DL_MIN_PARALLEL and conns > 1 and parallel_download(url, part, total, label, conns, dst.name)):+ _claim_part(part, url)download_single(url, part, total, label)if total and part.stat().st_size != total:fail(f"could not finish downloading {dst.name}: {part.stat().st_size:,} bytes on disk, the server says {total:,}","check your internet connection and run it again (the download resumes where it stopped)")- part.replace(dst)- meta = ms_meta(*ms) if ms is not None else None+ try:+ atomic.replace(part, dst) # waits a moment for a virus scanner or indexer that holds the file+ except OSError as e:+ fail(f"cannot finish {dst.name}: {e}", "close any program that has the file open and run setup again")+ for side in (".chunks", ".src"):+ part.with_name(part.name + side).unlink(missing_ok=True)if pin:- verify_sha256(dst, *pin) # the fixed hash: a wrong file is deleted and setup stops+ verify_sha256(dst, *pin, repair=again) # the fixed hash: a wrong file is repaired, else deleted and setup stopselif meta is not None and meta[1]:- verify_sha256(dst, meta[0], meta[1]) # ModelScope's published hash; kept in the finish mark+ verify_sha256(dst, meta[0], meta[1], repair=again) # ModelScope's published hash; kept in the finish markelse:if ms is not None:warn(f"{what or dst.name}: ModelScope publishes no SHA-256 for it: the file is not verified")@@ -2526,19 +2766,7 @@ def gguf_dir_problem(folder: Path, first: Path, fam: dict, model: str) -> tuplereturn f"{folder} has no {fam['title']} {model} file", hint-def verify_sha256(s: Path, size: int, sha: str, own: bool = False) -> None:- """A shard's size and SHA-256 against the pinned values (the Unsloth file); the result is kept in its finish mark,- so the ~5 minutes of hashing 111 GB happen once. A wrong file is deleted, so the next run downloads it again."""- m = s.with_name(s.name + ".done")- if m.exists() and f"sha256 {sha}" in m.read_text(encoding="utf-8", errors="replace"):- return- have = s.stat().st_size if s.exists() else -1- if have != size:- if own: # --gguf-dir: the person's own file is never deleted by setup- fail(f"{s.name} is {have:,} bytes; the published file is {size:,}",- "it may be cut short or another version; your file was left as it is. Copy the whole published file into the folder")- fail(f"{s.name} is {have:,} bytes, not {size:,}", "delete it and run setup again (the download restarts)")- say(f" checking {s.name} (SHA-256, {size / 1e9:.1f} GB) ...")+def _hash_file(s: Path, size: int) -> str:h = hashlib.sha256()seen, last = 0, 0.0with open(s, "rb") as f:@@ -2553,13 +2781,40 @@ def verify_sha256(s: Path, size: int, sha: str, own: bool = False) -> None:print(f"\r checking {s.name}: {seen / 1e9:6.2f} / {size / 1e9:.2f} GB ({100 * seen // size}%) ", end="", flush=True)if seen >= (1 << 30):print()- if h.hexdigest() != sha:+ return h.hexdigest()+++def verify_sha256(s: Path, size: int, sha: str, own: bool = False, repair=None) -> None:+ """A shard's size and SHA-256 against the pinned values (the Unsloth file); the result is kept in its finish mark,+ so the ~5 minutes of hashing 111 GB happen once. A wrong file is deleted, so the next run downloads it again.+ `repair` (a downloaded file's own re-reader, see repair_download): the wrong file is first compared with the server+ piece by piece and only the damaged pieces are written again, then it is checked a second time."""+ m = s.with_name(s.name + ".done")+ if m.exists() and f"sha256 {sha}" in m.read_text(encoding="utf-8", errors="replace"):+ return+ have = s.stat().st_size if s.exists() else -1+ if have != size:+ if own: # --gguf-dir: the person's own file is never deleted by setup+ fail(f"{s.name} is {have:,} bytes; the published file is {size:,}",+ "it may be cut short or another version; your file was left as it is. Copy the whole published file into the folder")+ fail(f"{s.name} is {have:,} bytes, not {size:,}", "delete it and run setup again (the download restarts)")+ say(f" checking {s.name} (SHA-256, {size / 1e9:.1f} GB) ...")+ digest = _hash_file(s, size)+ if digest != sha and repair is not None and not own:+ say(f" Some parts of {s.name} were damaged; downloading only those parts again.")+ res = repair()+ if res is not None:+ say(f" {res[0]} of {res[1]} parts were different and have been written again; checking the whole file again ...")+ digest = _hash_file(s, size)+ else:+ say(" The damaged parts could not be found this way.")+ if digest != sha:if own:- fail(f"{s.name} does not match the published file (SHA-256 {h.hexdigest()}, expected {sha})",+ fail(f"{s.name} does not match the published file (SHA-256 {digest}, expected {sha})","it may be damaged or another version; your file was left as it is. Copy the whole published file into the folder")s.unlink(missing_ok=True)m.unlink(missing_ok=True)- fail(f"{s.name} has the wrong SHA-256 ({h.hexdigest()}, expected {sha}): deleted",+ fail(f"{s.name} has the wrong SHA-256 ({digest}, expected {sha}): deleted","run setup again to download it again")mark(s, f"sha256 {sha}")diff --git a/strata_control/transfer.py b/strata_control/transfer.pyindex 256877319..e416b3777 100644--- a/strata_control/transfer.py+++ b/strata_control/transfer.py@@ -15,6 +15,7 @@ import hashlibimport http.clientimport ipaddressimport os+import reimport threadingimport email.utilsimport datetime as _dt@@ -228,8 +229,11 @@ def _transfer(location, partial: Path, have: int, size: int, progress, cancel, theaders["Range"] = f"bytes={have}-"request = urllib.request.Request(location, headers=headers)with _opener.open(request, timeout=timeout) as response:- if have and response.status != 206:- have = done = 0 # the server ignored the range: start over, never append to a prefix+ if have and not _resumes_at(response, have, size):+ if response.status == 206: # another range: its bytes are not the file's, ask again from the start+ partial.unlink(missing_ok=True)+ return _transfer(location, partial, 0, size, progress, cancel, timeout)+ have = done = 0 # the server ignored the range and sent the whole file: start over, never append to a prefix_copy(response, partial, "wb" if not have else "ab", done, size, progress, cancel)else:path = _local_path(location)@@ -252,6 +256,17 @@ def _transfer(location, partial: Path, have: int, size: int, progress, cancel, tresumable=True) from None+_CONTENT_RANGE = re.compile(r"\s*bytes (\d+)-(\d+)/(\d+|\*)\s*$")+++def _resumes_at(response, have: int, size: int) -> bool:+ """Whether a resumed answer is a 206 whose Content-Range starts at byte `have` of a file of `size` bytes."""+ if response.status != 206:+ return False+ m = _CONTENT_RANGE.match(response.headers.get("Content-Range") or "")+ return bool(m) and int(m.group(1)) == have and (m.group(3) == "*" or int(m.group(3)) == size)++def _copy(source, partial: Path, mode: str, done: int, size: int, progress, cancel) -> None:with partial.open(mode) as out:while True:diff --git a/tests/terminal/test_download_integrity.py b/tests/terminal/test_download_integrity.pynew file mode 100644index 000000000..64b27654e--- /dev/null+++ b/tests/terminal/test_download_integrity.py@@ -0,0 +1,443 @@+"""0.2.2: a model download never leaves wrong bytes behind, and a file with a wrong SHA-256 is repaired, not thrown away.++A user's 39.8 GB shard arrived with the wrong SHA-256 on 0.2.1 (16 connections, several pieces per request). These tests run+setup.download() against a local HTTP server that misbehaves in each way a server, a CDN or a disk can: a 200 for a ranged+request, a 206 for another range or another total, a body that ends early, another version of the file part-way, a changed+ETag, a .part written from another mirror, no piece list, a full disk, a raw write that takes fewer bytes than given, and+bytes that are wrong in the first download (the repair reads the file again and writes only the pieces that differ).+Pieces are 64 KiB here instead of 32 MiB, so a few MiB stand for the 40 GB."""++from __future__ import annotations++import contextlib+import hashlib+import http.server+import io+import json+import random+import re+import shutil+import tempfile+import threading+import unittest+from pathlib import Path+from unittest import mock++import setup+from strata_control import transfer++PIECE = 64 << 10+SIZE = 40 * PIECE + 12345 # 41 pieces, the last one short+DATA = random.Random(7).randbytes(SIZE)+SHA = hashlib.sha256(DATA).hexdigest()+++class Behavior:+ """What the server does with request number n (1 = the first). `plan` returns overrides for that answer."""++ def __init__(self, plan=None, data=DATA, etag='"v1"'):+ self.plan = plan or (lambda n, start, end: {})+ self.data = data+ self.etag = etag+ self.lock = threading.Lock()+ self.count = 0+ self.served = [] # (start, end) of every ranged answer+ self.once = set()++ def next(self):+ with self.lock:+ self.count += 1+ return self.count+++class Handler(http.server.BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.0" # one request per connection: ending a body early is a reset++ def log_message(self, *a):+ pass++ def do_HEAD(self):+ beh = self.server.beh+ self.send_response(200)+ self.send_header("Content-Length", str(len(beh.data)))+ self.send_header("Accept-Ranges", "bytes")+ self.end_headers()++ def do_GET(self):+ beh = self.server.beh+ n = beh.next()+ total = len(beh.data)+ m = re.match(r"bytes=(\d+)-(\d*)", self.headers.get("Range") or "")+ start = int(m.group(1)) if m else 0+ end = min(int(m.group(2)), total - 1) if m and m.group(2) else total - 1+ if start >= total:+ self.send_response(416)+ self.end_headers()+ return+ o = beh.plan(n, start, end) or {}+ if o.get("status") == 404 or o.get("status") == 503:+ self.send_response(o["status"])+ self.send_header("Content-Length", "0")+ self.end_headers()+ return+ status = o.get("status", 206 if m else 200)+ b0, b1 = o.get("body", (start, end))+ body = bytearray(beh.data if status == 200 else beh.data[b0:b1 + 1])+ for off in o.get("corrupt", ()):+ lo = 0 if status == 200 else b0+ if lo <= off < lo + len(body):+ body[off - lo] ^= 0xFF+ self.send_response(status)+ self.send_header("Content-Length", str(o.get("cl", len(body))))+ if status == 206:+ self.send_header("Content-Range", o.get("cr", f"bytes {start}-{end}/{total}"))+ self.send_header("ETag", o.get("etag", beh.etag))+ self.end_headers()+ with beh.lock:+ beh.served.append((start, end))+ cut = o.get("cut")+ try:+ self.wfile.write(bytes(body[:cut] if cut is not None else body))+ except OSError:+ pass+++class Server(http.server.ThreadingHTTPServer):+ daemon_threads = True++ def handle_error(self, request, client_address):+ pass+++class DownloadCase(unittest.TestCase):+ def setUp(self):+ self.tmp = Path(tempfile.mkdtemp(prefix="dlint"))+ self.addCleanup(shutil.rmtree, self.tmp, True)+ patches = [mock.patch.object(setup, "DL_CHUNK", PIECE), mock.patch.object(setup, "DL_MIN_PARALLEL", PIECE * 4),+ mock.patch.object(setup, "DL_BACKOFF", 0.0), mock.patch.object(setup, "DL_SAVE_EVERY", 0.0),+ mock.patch.object(setup, "DL_ROUNDS", 3), mock.patch.object(setup.time, "sleep", lambda s: None),+ mock.patch.object(setup, "file_pin", lambda url: (SIZE, SHA)),+ mock.patch.dict("os.environ", {"STRATA_DL_CONNECTIONS": "4", "STRATA_SOURCE": "huggingface"})]+ for p in patches:+ p.start()+ self.addCleanup(p.stop)+ self.dst = self.tmp / "model.gguf"+ self.part = self.tmp / "model.gguf.part"++ def serve(self, beh):+ srv = Server(("127.0.0.1", 0), Handler)+ srv.beh = beh+ threading.Thread(target=srv.serve_forever, daemon=True).start()+ self.addCleanup(srv.server_close)+ self.addCleanup(srv.shutdown)+ self.beh = beh+ return f"http://127.0.0.1:{srv.server_address[1]}/m/model.gguf"++ def run_download(self, url, expect_exit=False):+ buf = io.StringIO()+ with contextlib.redirect_stdout(buf):+ if expect_exit:+ with self.assertRaises(SystemExit):+ setup.download(url, self.dst)+ else:+ setup.download(url, self.dst)+ self.out = buf.getvalue()+ return self.out++ def assertGood(self):+ self.assertTrue(self.dst.is_file(), self.out)+ self.assertEqual(hashlib.sha256(self.dst.read_bytes()).hexdigest(), SHA)+ self.assertFalse(self.part.exists())+ self.assertFalse(self.tmp.joinpath("model.gguf.part.chunks").exists())+++class Misbehaving(DownloadCase):+ def test_clean_download(self):+ self.run_download(self.serve(Behavior()))+ self.assertGood()++ def test_ranged_requests_answered_with_200_use_one_stream(self):+ self.run_download(self.serve(Behavior(lambda n, s, e: {"status": 200})))+ self.assertGood()++ def test_ranges_stop_being_honoured_part_way(self):+ # the probe and the first pieces are right; later requests get the whole file with a 200+ self.run_download(self.serve(Behavior(lambda n, s, e: {"status": 200} if n > 6 else {})))+ self.assertGood()++ def test_206_for_another_range_is_not_written(self):+ # Content-Range names a range that starts later than the one asked for, from the 5th request on+ self.run_download(self.serve(Behavior(lambda n, s, e: {"cr": f"bytes {s + 7}-{e}/{SIZE}", "body": (s + 7, e)}+ if n > 4 and s > 0 else {})))+ self.assertGood()++ def test_206_with_another_total_is_not_written(self):+ self.run_download(self.serve(Behavior(lambda n, s, e: {"cr": f"bytes {s}-{e}/{SIZE + 1}"} if n > 4 else {})))+ self.assertGood()++ def test_206_with_a_longer_body_than_asked(self):+ # Content-Length does not fit the range: tried again, then right+ self.run_download(self.serve(Behavior(lambda n, s, e: {"cl": e - s + 5} if n in (4, 5, 9) else {})))+ self.assertGood()++ def test_connection_reset_in_the_middle_of_pieces(self):+ # every 3rd answer ends after a random part of its body; the piece it broke in is written again from its start+ rnd = random.Random(3)+ self.run_download(self.serve(Behavior(lambda n, s, e: {"cut": rnd.randrange(1, max(2, e - s))} if n % 3 == 0 else {})))+ self.assertGood()++ def test_an_answer_cut_after_a_whole_number_of_blocks_still_ends_in_the_right_bytes(self):+ self.run_download(self.serve(Behavior(lambda n, s, e: {"cut": PIECE + 17} if n in (3, 4, 5, 6) else {})))+ self.assertGood()++ def test_one_answer_from_another_version_is_asked_again(self):+ self.run_download(self.serve(Behavior(lambda n, s, e: {"etag": '"v2"'} if n in (4, 7) else {})))+ self.assertGood()++ def test_another_version_for_good_stops_and_leaves_no_file(self):+ out = self.run_download(self.serve(Behavior(lambda n, s, e: {"etag": '"v2"'} if n > 3 else {})), expect_exit=True)+ self.assertFalse(self.dst.exists())++ def test_the_file_changing_between_resolves_is_noticed(self):+ src = setup._Source("http://x/y", SIZE)+ src.etag = "aaa"++ class R:+ status = 206+ headers = {"Content-Range": f"bytes 0-0/{SIZE}", "ETag": '"bbb"'}++ def __enter__(self):+ return self++ def __exit__(self, *a):+ return False++ def read(self, n):+ return b"x"++ def geturl(self):+ return "http://x/y"++ with mock.patch.object(setup, "_urlopen", lambda *a, **k: R()):+ with self.assertRaises(setup._Changed):+ src.resolve()++ def test_server_error_part_way_keeps_pieces_and_a_second_run_finishes(self):+ url = self.serve(Behavior(lambda n, s, e: {"status": 503} if n > 12 else {}))+ self.run_download(url, expect_exit=True)+ self.assertTrue(self.part.exists())+ self.assertTrue(self.tmp.joinpath("model.gguf.part.chunks").exists())+ first_run = len(self.beh.served)+ self.beh.plan = lambda n, s, e: {} # the same address, now answering+ self.beh.served.clear()+ self.run_download(url)+ self.assertGood()+ fetched = sum(e - s + 1 for s, e in self.beh.served if e > s) # without the one-byte probes+ self.assertLess(fetched, SIZE, "the second run took the whole file again instead of resuming")+ self.assertGreater(first_run, 1)++ def test_two_interrupted_runs_then_a_good_one(self):+ url = self.serve(Behavior())+ for limit in (6, 6):+ self.beh.count = 0+ self.beh.plan = lambda n, s, e, limit=limit: {"status": 503} if n > limit else {}+ self.run_download(url, expect_exit=True)+ self.beh.plan = lambda n, s, e: {}+ self.run_download(url)+ self.assertGood()++ def test_a_part_written_from_another_mirror_is_dropped(self):+ url = self.serve(Behavior())+ self.part.write_bytes(b"\xaa" * (PIECE * 10))+ self.tmp.joinpath("model.gguf.part.chunks").write_text(+ json.dumps({"v": 1, "total": SIZE, "chunk": PIECE, "done": [[0, 10]]}), encoding="utf-8")+ self.tmp.joinpath("model.gguf.part.src").write_text(+ json.dumps({"url": "https://www.modelscope.cn/models/x/resolve/master/model.gguf", "etag": None}), encoding="utf-8")+ self.run_download(url)+ self.assertGood() # the 0xaa pieces were not trusted++ def test_a_part_of_another_version_is_dropped(self):+ url = self.serve(Behavior())+ self.part.write_bytes(b"\xaa" * (PIECE * 10))+ self.tmp.joinpath("model.gguf.part.chunks").write_text(+ json.dumps({"v": 1, "total": SIZE, "chunk": PIECE, "done": [[0, 10]]}), encoding="utf-8")+ self.tmp.joinpath("model.gguf.part.src").write_text(json.dumps({"url": url, "etag": '"old"'}), encoding="utf-8")+ self.run_download(url)+ self.assertGood()++ def test_without_a_writable_piece_list_the_file_is_not_written_in_parts(self):+ url = self.serve(Behavior())+ with mock.patch.object(setup, "_save_pieces", return_value=False):+ self.run_download(url)+ self.assertGood()++ def test_a_stale_piece_list_never_names_bytes_the_part_lacks(self):+ self.part.write_bytes(b"\x00" * (PIECE * 3))+ self.tmp.joinpath("model.gguf.part.chunks").write_text(+ json.dumps({"v": 1, "total": SIZE, "chunk": PIECE, "done": [[0, 30]]}), encoding="utf-8")+ self.assertEqual(setup._load_pieces(self.part, SIZE), {0, 1, 2})++ def test_full_disk_stops_with_a_plain_message_and_keeps_the_part(self):+ url = self.serve(Behavior())++ def full(f, pos, data):+ raise OSError(28, "No space left on device")++ with mock.patch.object(setup, "_write_all", full):+ out = self.run_download(url, expect_exit=True)+ self.assertIn("cannot write", out)+ self.assertIn("free some disk space", out)+ self.assertFalse(self.dst.exists())++ def test_a_raw_write_that_takes_fewer_bytes_is_finished(self):+ class Slow:+ def __init__(self):+ self.buf = bytearray(100)+ self.pos = 0++ def seek(self, p):+ self.pos = p++ def write(self, v):+ n = min(7, len(v))+ self.buf[self.pos:self.pos + n] = bytes(v[:n])+ self.pos += n+ return n++ f = Slow()+ setup._write_all(f, 10, bytes(range(60)))+ self.assertEqual(bytes(f.buf[10:70]), bytes(range(60)))++ def test_a_raw_write_that_takes_nothing_is_an_error(self):+ class Zero:+ def seek(self, p):+ pass++ def write(self, v):+ return 0++ with self.assertRaises(OSError):+ setup._write_all(Zero(), 0, b"abc")++ def test_single_stream_resume_ignores_a_206_from_the_wrong_offset(self):+ url = self.serve(Behavior(lambda n, s, e: {"cr": f"bytes {s + 5}-{e}/{SIZE}", "body": (s + 5, e)} if s > 0 else {}))+ self.part.write_bytes(DATA[:PIECE * 3])+ with mock.patch.dict("os.environ", {"STRATA_DL_CONNECTIONS": "1"}):+ self.run_download(url)+ self.assertGood()++ def test_single_stream_resume_ignores_a_200(self):+ url = self.serve(Behavior(lambda n, s, e: {"status": 200}))+ self.part.write_bytes(DATA[:PIECE * 3])+ with mock.patch.dict("os.environ", {"STRATA_DL_CONNECTIONS": "1"}):+ self.run_download(url)+ self.assertGood()++ def test_finished_file_without_a_mark_is_checked_not_marked(self):+ self.dst.write_bytes(DATA[:-1] + b"\x00")+ url = self.serve(Behavior())+ self.run_download(url)+ self.assertIn("Some parts of model.gguf were damaged", self.out)+ self.assertGood()+ self.assertTrue(self.tmp.joinpath("model.gguf.done").exists())+++class Repair(DownloadCase):+ def test_wrong_bytes_in_two_pieces_are_found_and_only_those_rewritten(self):+ bad = [5 * PIECE + 100, 30 * PIECE + 7]+ seen = set()++ def plan(n, s, e):+ hit = [o for o in bad if s <= o <= e and o not in seen]+ seen.update(hit)+ return {"corrupt": hit} if hit else {}++ out = self.run_download(self.serve(Behavior(plan)))+ self.assertGood()+ self.assertIn("Some parts of model.gguf were damaged; downloading only those parts again", out)+ self.assertIn("2 of 41 parts were different", out)+ self.assertTrue(self.tmp.joinpath("model.gguf.done").exists())++ def test_one_flipped_byte_in_the_last_short_piece(self):+ bad = [SIZE - 3]+ seen = set()++ def plan(n, s, e):+ hit = [o for o in bad if s <= o <= e and o not in seen]+ seen.update(hit)+ return {"corrupt": hit} if hit else {}++ out = self.run_download(self.serve(Behavior(plan)))+ self.assertGood()+ self.assertIn("1 of 41 parts were different", out)++ def test_a_repair_that_loses_the_connection_part_way_still_ends_right(self):+ bad = [3 * PIECE + 1]+ seen = set()++ def plan(n, s, e):+ hit = [o for o in bad if s <= o <= e and o not in seen]+ seen.update(hit)+ o = {"corrupt": hit} if hit else {}+ if seen and n % 4 == 0:+ o["cut"] = 5000+ return o++ self.run_download(self.serve(Behavior(plan)))+ self.assertGood()++ def test_a_server_that_keeps_serving_the_wrong_file_ends_with_the_file_deleted(self):+ wrong = bytearray(DATA)+ wrong[1000] ^= 0xFF+ url = self.serve(Behavior(data=bytes(wrong)))+ out = self.run_download(url, expect_exit=True)+ self.assertIn("wrong SHA-256", out)+ self.assertFalse(self.dst.exists())+ self.assertFalse(self.tmp.joinpath("model.gguf.done").exists())++ def test_the_repair_is_off_for_a_file_the_person_owns(self):+ self.dst.write_bytes(b"x" * SIZE)+ called = []+ with contextlib.redirect_stdout(io.StringIO()):+ with self.assertRaises(SystemExit):+ setup.verify_sha256(self.dst, SIZE, SHA, own=True, repair=lambda: called.append(1))+ self.assertEqual(called, [])+ self.assertTrue(self.dst.exists())+++class UpdateTransfer(unittest.TestCase):+ """strata_control/transfer.py (update archives): a resumed answer is appended only when it starts at the byte asked for."""++ def setUp(self):+ self.tmp = Path(tempfile.mkdtemp(prefix="dlint"))+ self.addCleanup(shutil.rmtree, self.tmp, True)++ def test_resume_check(self):+ class R:+ def __init__(self, status, cr):+ self.status = status+ self.headers = {"Content-Range": cr} if cr else {}++ self.assertTrue(transfer._resumes_at(R(206, "bytes 100-999/1000"), 100, 1000))+ self.assertFalse(transfer._resumes_at(R(206, "bytes 0-999/1000"), 100, 1000))+ self.assertFalse(transfer._resumes_at(R(206, "bytes 100-999/2000"), 100, 1000))+ self.assertFalse(transfer._resumes_at(R(206, ""), 100, 1000))+ self.assertFalse(transfer._resumes_at(R(200, "bytes 100-999/1000"), 100, 1000))++ def test_a_206_from_the_wrong_offset_restarts_the_file(self):+ data = DATA[:200000]+ beh = Behavior(lambda n, s, e: {"cr": f"bytes {s + 9}-{e}/{len(data)}", "body": (s + 9, e)} if s > 0 else {}, data=data)+ srv = Server(("127.0.0.1", 0), Handler)+ srv.beh = beh+ threading.Thread(target=srv.serve_forever, daemon=True).start()+ self.addCleanup(srv.server_close)+ self.addCleanup(srv.shutdown)+ part = self.tmp / "a.part"+ part.write_bytes(data[:50000])+ got = transfer.download(f"http://127.0.0.1:{srv.server_address[1]}/a", part, size=len(data),+ sha256=hashlib.sha256(data).hexdigest())+ self.assertEqual(got.read_bytes(), data)+++if __name__ == "__main__":+ unittest.main()