Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Downloads: 16 connections, several pieces per request, ModelScope length probe, faster MTP fetch
Niko1221e2071c2f8b12
diff --git a/setup.py b/setup.pyindex 328aa5b1f..c7a2d32aa 100644--- a/setup.py+++ b/setup.py@@ -1913,12 +1913,14 @@ def drop_archive(z: Path) -> None:# each connection): download() takes the file in DL_CHUNK pieces over several connections, writes each piece at its own offset,# and keeps the finished pieces in <name>.part.chunks so a stopped download continues where it was. Anything that is not a# ranged file of known length (no Range support, no length, a small file) is one stream, as before.-DL_CONNECTIONS = 8 # STRATA_DL_CONNECTIONS=1 restores one stream (for testers)-DL_CHUNK = 32 << 20 # bytes per ranged request; a failed request loses at most this much+DL_CONNECTIONS = 16 # STRATA_DL_CONNECTIONS=1 restores one stream (for testers)+DL_CHUNK = 32 << 20 # bytes per piece: what the piece list records and what a failed request loses at most+DL_RUN = 4 # pieces one request asks for (4 x 32 MiB): a new connection per piece spends its first seconds+ # in TCP slow start (measured: 8 connections 66 MB/s with a request per piece, 73 with 128 MiB)DL_MIN_PARALLEL = 256 << 20 # smaller files are one stream-DL_CHUNK_TRIES = 5 # tries per piece (1, 2, 4, 8 s apart) before the round is given up and run again+DL_CHUNK_TRIES = 5 # tries per request (1, 2, 4, 8 s apart) before the round is given up and run againDL_ROUNDS = 30 # rounds (10 s apart) before the download is given up, like the 30 tries of one stream-DL_BACKOFF = 1.0 # seconds before the second try of a piece, doubled each time (tests set 0)+DL_BACKOFF = 1.0 # seconds before the second try of a request, doubled each time (tests set 0)DL_SAVE_EVERY = 5.0 # seconds between writes of the progress file@@ -1994,8 +1996,8 @@ class _Source:class _Pieces:"""The shared state of one parallel download: finished pieces, bytes of the pieces in flight, the stop flags."""- def __init__(self, total: int, done: set):- self.total, self.n = total, -(-total // DL_CHUNK)+ def __init__(self, total: int, done: set, conns: int = 1):+ self.total, self.n, self.conns = total, -(-total // DL_CHUNK), connsself.done, self.flight = set(done), {}self.lock = threading.Lock()self.stop = threading.Event() # the whole download stops (Ctrl-C, a fatal error)@@ -2104,16 +2106,32 @@ def _single_ready(part: Path):side.unlink(missing_ok=True)-def _fetch_piece(src: _Source, st: _Pieces, f, i: int):- start = i * DL_CHUNK- end = start + st.size(i) - 1+def _take_run(st: _Pieces, todo: list) -> list:+ """The next request's pieces: up to DL_RUN neighbours from the front of `todo`; fewer near the end, so the last pieces+ are spread over every connection and no single slow one holds the finish."""+ with st.lock:+ if not todo:+ return []+ n = max(1, min(DL_RUN, len(todo) // (2 * st.conns)))+ run = [todo.pop(0)]+ while len(run) < n and todo and todo[0] == run[-1] + 1:+ run.append(todo.pop(0))+ return run+++def _fetch_run(src: _Source, st: _Pieces, f, run: list):+ """One ranged request for the neighbouring pieces in `run`, written at their offsets; each piece counts as finished+ when its last byte is down. A connection that breaks continues from the byte it reached, not from the piece start."""+ start = run[0] * DL_CHUNK+ end = min((run[-1] + 1) * DL_CHUNK, st.total) - 1pos, tries = start, 0with st.lock:- st.flight[i] = 0+ for i in run:+ st.flight[i] = 0while True:if st.stop.is_set():return- url, err, wait = None, None, DL_BACKOFF * 2 ** tries+ url, err, got, wait, ra = None, None, False, None, 0.0try:url = src.get()req = urllib.request.Request(url, headers={"User-Agent": "strata-setup", "Range": f"bytes={pos}-{end}"})@@ -2123,7 +2141,9 @@ def _fetch_piece(src: _Source, st: _Pieces, f, i: int):while pos <= end:if st.stop.is_set():return- b = r.read(min(1 << 20, end + 1 - pos))+ cur = pos // DL_CHUNK+ cur_end = min((cur + 1) * DL_CHUNK, st.total) # one past the piece's last byte+ b = r.read(min(1 << 20, cur_end - pos))if not b:raise IOError("the connection closed early")try:@@ -2132,8 +2152,16 @@ def _fetch_piece(src: _Source, st: _Pieces, f, i: int):except OSError as e:raise _DiskError(str(e)) from epos += len(b)- st.add(i, len(b))- st.finish(i)+ got = True+ st.add(cur, len(b))+ if pos == cur_end:+ st.finish(cur)+ if pos <= end and st.abort_round.is_set(): # a piece failed for good: stop at a piece edge+ with st.lock:+ for i in run:+ if i not in st.done and st.flight.get(i) == 0:+ st.flight.pop(i, None)+ returnreturnexcept (_NoRange, _DiskError):raise@@ -2149,26 +2177,25 @@ def _fetch_piece(src: _Source, st: _Pieces, f, i: int):passelif e.code in (429, 503):try:- wait = max(wait, min(float(e.headers.get("Retry-After") or 0), 60))+ ra = min(float(e.headers.get("Retry-After") or 0), 60)except (TypeError, ValueError):passexcept (OSError, http.client.HTTPException) as e:err = e- tries += 1+ tries = 1 if got else tries + 1 # a try that moved forward is not a failed oneif tries >= DL_CHUNK_TRIES:raise _PieceFailed(str(err))- st.stop.wait(wait)+ st.stop.wait(max(DL_BACKOFF * 2 ** (tries - 1), ra) if wait is None else wait)def _piece_worker(src: _Source, st: _Pieces, todo: list, part: Path):try:with open(part, "r+b", buffering=0) as f:while not (st.stop.is_set() or st.abort_round.is_set()):- with st.lock:- if not todo:- return- i = todo.pop(0)- _fetch_piece(src, st, f, i)+ run = _take_run(st, todo)+ if not run:+ return+ _fetch_run(src, st, f, run)except _PieceFailed as e:st.errors.append(e)st.abort_round.set()@@ -2191,7 +2218,7 @@ def parallel_download(url: str, part: Path, total: int, label: str, conns: int,except (OSError, http.client.HTTPException):return False # the one-stream path has its own retries and messagesdone = _load_pieces(part, total)- st = _Pieces(total, done)+ st = _Pieces(total, done, conns)part.parent.mkdir(parents=True, exist_ok=True)if not part.exists():part.touch()@@ -2292,6 +2319,18 @@ def download_single(url: str, part: Path, total: int, label: str):time.sleep(10)+def range_total(url: str) -> int:+ """The file's length from a one-byte ranged GET (Content-Range: bytes 0-0/N), for a server whose HEAD gives none+ (ModelScope's: #1915); 0 when it does not say. With the length known the file comes over several connections."""+ try:+ req = urllib.request.Request(url, headers={"User-Agent": "strata-setup", "Range": "bytes=0-0"})+ with _urlopen(req, timeout=60) as r:+ m = re.match(r"\s*bytes 0-0/(\d+)\s*$", r.headers.get("Content-Range") or "")+ return int(m.group(1)) if r.status == 206 and m else 0+ except (OSError, http.client.HTTPException):+ return 0++def download(url, dst: Path, what=None):"""Resumable HTTP(S) download with a progress line; `file://` and plain paths are copied (tests, mirrors).A finished file gets a <name>.done mark, so a later run skips it without asking the server."""@@ -2343,6 +2382,8 @@ def download(url, dst: Path, what=None):fail(f"cannot reach {url.split('/')[2]} ({e})","check your internet connection and run it again" + source_hint(url))time.sleep(5)+ if not total:+ total = range_total(url)if pin and total and total != pin[0]: # not the pinned file: stop before the download, not afterfail(f"{what or dst.name} is {total:,} bytes on {url.split('/')[2]}, not the pinned {pin[0]:,}","update Strata (the file there has changed since this version pinned it)")diff --git a/tools/mtp_fetch.py b/tools/mtp_fetch.pyindex 04d8f8b88..b4a44dfd5 100644--- a/tools/mtp_fetch.py+++ b/tools/mtp_fetch.py@@ -24,6 +24,7 @@ import jsonimport osimport structimport sys+import threadingimport timeimport urllib.errorimport urllib.request@@ -115,20 +116,41 @@ SHA256 = {}+# Hugging Face answers a /resolve/ address with a redirect to a signed CDN address (valid for about an hour). Every range of a+# tensor used to go through that redirect and a fresh connection to it; the CDN address of the first answer is reused now+# (and forgotten after any error, so the next try asks again).+CDN_TTL = 600+_cdn = {}+_cdn_lock = threading.Lock()+++def _final(url):+ with _cdn_lock:+ hit = _cdn.get(url)+ return hit[0] if hit and hit[1] > time.monotonic() else url++def get(url, start=None, end=None, retries=4):for attempt in range(retries):try:- req = urllib.request.Request(url, headers={"User-Agent": "strata-mtp-fetch"})+ used = _final(url)+ req = urllib.request.Request(used, headers={"User-Agent": "strata-mtp-fetch"})if start is not None:req.add_header("Range", "bytes=%d-%d" % (start, end))with urllib.request.urlopen(req, timeout=120) as r:if start is not None:check_range(r.status, r.headers.get("Content-Range"), start, end)data = r.read()+ landed = getattr(r, "geturl", lambda: url)()if start is not None and len(data) != end - start + 1:raise IOError("short range read: %d of %d" % (len(data), end - start + 1))+ if used == url and landed and landed != url:+ with _cdn_lock:+ _cdn[url] = (landed, time.monotonic() + CDN_TTL)return dataexcept Exception as e: # network errors are retried, then surfaced+ with _cdn_lock:+ _cdn.pop(url, None)if attempt == retries - 1:raisetime.sleep(2 ** attempt)@@ -136,11 +158,11 @@ def get(url, start=None, end=None, retries=4):def connections():- """Ranges read at once: STRATA_DL_CONNECTIONS (the setting of setup's downloads), else 8; 1 reads them one by one."""+ """Ranges read at once: STRATA_DL_CONNECTIONS (the setting of setup's downloads), else 16; 1 reads them one by one."""try:- n = int((os.environ.get("STRATA_DL_CONNECTIONS") or "").strip() or 8)+ n = int((os.environ.get("STRATA_DL_CONNECTIONS") or "").strip() or 16)except ValueError:- n = 8+ n = 16return max(1, min(n, 32))@@ -158,10 +180,13 @@ def ranged(url, start, last, chunk, workers):returntodo = iter(spans)window = collections.deque()+ # the ranges are handed on in order, so one slow range holds the oldest slot; the connections that are done go on with+ # the next ranges meanwhile (up to `depth` ahead, at most ~512 MiB in memory) instead of waiting for it+ depth = max(workers, min(3 * workers, (512 << 20) // max(chunk, 1)))with concurrent.futures.ThreadPoolExecutor(min(workers, len(spans))) as ex:for a, b in todo:window.append(ex.submit(get, url, a, b))- if len(window) >= workers:+ if len(window) >= depth:breakwhile window:data = window.popleft().result()@@ -225,8 +250,11 @@ def inventory(out):index = json.loads(get(REPO + "model.safetensors.index.json"))["weight_map"]mtp = {k: v for k, v in index.items() if k.startswith("mtp.")}rows, total = [], 0- for shard in sorted(set(mtp.values())):- base, header = shard_header(shard)+ shards = sorted(set(mtp.values()))+ # one small request pair per shard (28 of them): side by side, not one after the other (28 s -> 4 s)+ with concurrent.futures.ThreadPoolExecutor(max(1, min(connections(), 8, len(shards)))) as ex:+ headers = list(ex.map(shard_header, shards))+ for shard, (base, header) in zip(shards, headers):for name, meta in header.items():if name in mtp and mtp[name] == shard:a, b = meta["data_offsets"]@@ -268,7 +296,7 @@ def fetch(out, only):os.makedirs(tdir, exist_ok=True)manifest = []workers = connections()- chunk = (32 if workers > 1 else 64) << 20+ chunk = (16 if workers > 1 else 64) << 20for r in rows:if only and only not in r["name"]:continuediff --git a/tools/test_mtp_fetch.py b/tools/test_mtp_fetch.pyindex cb66527c7..bf2476506 100644--- a/tools/test_mtp_fetch.py+++ b/tools/test_mtp_fetch.py@@ -145,7 +145,60 @@ class Ranges(FetchCase):with mock.patch.dict(M.os.environ, {"STRATA_DL_CONNECTIONS": "1"}):self.assertEqual(M.connections(), 1)with mock.patch.dict(M.os.environ, {"STRATA_DL_CONNECTIONS": ""}):- self.assertEqual(M.connections(), 8)+ self.assertEqual(M.connections(), 16)++ def test_the_cdn_address_is_reused_and_forgotten_after_an_error(self):+ """HF redirects /resolve/ to a signed CDN address: only the first range goes through the redirect (the rest of a+ tensor's ranges use the CDN address), and any error makes the next try ask the original address again."""+ asked = []++ def opener(req, timeout=None):+ asked.append(req.full_url)+ a, b = (int(x) for x in req.headers["Range"].split("=")[1].split("-"))+ r = Response(bytes(b - a + 1), 206, {"Content-Range": "bytes %d-%d/1000" % (a, b)})+ r.geturl = lambda: "https://cdn.example/x?sig=1"+ return r++ M._cdn.clear()+ with mock.patch.object(M.urllib.request, "urlopen", opener):+ for a in (0, 100, 200):+ M.get("https://hf.example/f", a, a + 99)+ self.assertEqual(asked, ["https://hf.example/f"] + ["https://cdn.example/x?sig=1"] * 2)+ asked.clear()+ fails = [True]++ def flaky(req, timeout=None):+ if fails.pop() if fails else False:+ raise IOError("expired")+ return opener(req, timeout)++ with mock.patch.object(M.urllib.request, "urlopen", flaky), mock.patch.object(M.time, "sleep", lambda s: None), contextlib.redirect_stderr(io.StringIO()):+ M.get("https://hf.example/f", 300, 399)+ self.assertEqual(asked, ["https://hf.example/f"]) # the cached address failed: the original is asked again+ M._cdn.clear()++ def test_a_slow_first_range_does_not_stop_the_ones_behind_it(self):+ import threading+ gate = threading.Event()+ started = []++ def get(url, a, b):+ started.append(a)+ if a == 0:+ gate.wait(5)+ return bytes(b - a + 1)++ def release(): # lets the first range go once the later ones have been started+ for _ in range(200):+ if len(started) >= 9:+ break+ M.time.sleep(0.01)+ gate.set()++ threading.Thread(target=release, daemon=True).start()+ with mock.patch.object(M, "get", get):+ list(M.ranged("u", 0, 40 * 1000 - 1, 1000, 4))+ self.assertGreaterEqual(len(started), 9) # four connections, nine ranges started before range 0 endeddef test_a_failing_range_stops_the_parallel_read(self):def get(url, a, b):diff --git a/tools/test_setup_download.py b/tools/test_setup_download.pyindex a14f14056..fc2e3236c 100644--- a/tools/test_setup_download.py+++ b/tools/test_setup_download.py@@ -142,7 +142,7 @@ def pin_of(data):@contextlib.contextmanagerdef small_pieces(**extra):"""Setup's piece size, size floor and waits scaled down to the test file."""- consts = dict(DL_CHUNK=CHUNK, DL_MIN_PARALLEL=0, DL_BACKOFF=0.0, DL_SAVE_EVERY=0.05, DL_CONNECTIONS=4)+ consts = dict(DL_CHUNK=CHUNK, DL_RUN=1, DL_MIN_PARALLEL=0, DL_BACKOFF=0.0, DL_SAVE_EVERY=0.05, DL_CONNECTIONS=4)consts.update(extra)with contextlib.ExitStack() as st:for k, v in consts.items():@@ -230,28 +230,38 @@ class NoRangeSupport(unittest.TestCase):self.assertEqual(srv.max_active, 1)self.assertEqual(files(d), [NAME, NAME + ".done"])- def test_a_server_without_a_length_is_read_in_one_stream(self):- with small_pieces(), Server(length=False) as srv, tempfile.TemporaryDirectory() as d:+ def test_a_server_without_any_length_is_read_in_one_stream(self):+ with small_pieces(), Server(length=False, ranges=False) as srv, tempfile.TemporaryDirectory() as d:code, out = run(srv, d, pin=None) # no length, so nothing to check a pin against firstself.assertIsNone(code, out)self.assertEqual((Path(d) / NAME).read_bytes(), DATA)self.assertEqual(srv.max_active, 1)+ def test_a_head_without_a_length_still_gets_the_connections(self):+ """ModelScope's HEAD has no Content-Length (#1915): the length comes from a one-byte ranged GET, not 'unknown'."""+ with small_pieces(), Server(length=False) as srv, tempfile.TemporaryDirectory() as d:+ code, out = run(srv, d)+ self.assertIsNone(code, out)+ self.assertEqual((Path(d) / NAME).read_bytes(), DATA)+ self.assertGreater(srv.max_active, 1)+ self.assertEqual(sorted(asked(srv)), list(range(41)))+ self.assertEqual(files(d), [NAME, NAME + ".done"])+def test_range_stopping_in_the_middle_falls_back_to_the_unbroken_start(self):def faults(path, start, end, nth):return Nonewith small_pieces(), Server(faults=faults) as srv, tempfile.TemporaryDirectory() as d:- real = setup._fetch_piece+ real = setup._fetch_runcalls = []- def fetch(src, st, f, i):- calls.append(i)+ def fetch(src, st, f, run):+ calls.append(run)if len(calls) == 6:raise setup._NoRange("HTTP 200")- return real(src, st, f, i)+ return real(src, st, f, run)- with mock.patch.object(setup, "_fetch_piece", fetch):+ with mock.patch.object(setup, "_fetch_run", fetch):code, out = run(srv, d)self.assertIsNone(code, out)self.assertEqual((Path(d) / NAME).read_bytes(), DATA) # the pin check passed: no gaps were kept@@ -471,3 +481,80 @@ class SignedAddress(unittest.TestCase):if __name__ == "__main__":unittest.main()+++class Runs(unittest.TestCase):+ """One request asks for several neighbouring pieces (DL_RUN): fewer connections to open, the same piece list."""++ def test_a_request_covers_several_pieces(self):+ with small_pieces(DL_RUN=4), Server() as srv, tempfile.TemporaryDirectory() as d:+ code, out = run(srv, d)+ self.assertIsNone(code, out)+ self.assertEqual((Path(d) / NAME).read_bytes(), DATA)+ self.assertEqual(files(d), [NAME, NAME + ".done"])+ spans = [(s, e) for s, e in srv.gets if (s, e) != (0, 0)]+ self.assertLess(len(spans), 41) # fewer requests than pieces+ self.assertTrue(any(e - s + 1 > CHUNK for s, e in spans), spans)+ self.assertTrue(all(e - s + 1 <= 4 * CHUNK for s, e in spans), spans)+ covered = sorted(i for s, e in spans for i in range(s // CHUNK, e // CHUNK + 1))+ self.assertEqual(covered, list(range(41))) # every piece once, none twice++ def test_the_last_pieces_are_asked_for_one_by_one(self):+ st = setup._Pieces(CHUNK * 100, set(), conns=4)+ todo = list(range(100))+ self.assertEqual(setup._take_run(st, todo), [0, 1, 2, 3])+ with mock.patch.object(setup, "DL_RUN", 4):+ self.assertEqual(setup._take_run(st, list(range(60, 100))), [60, 61, 62, 63])+ self.assertEqual(setup._take_run(st, list(range(93, 100))), [93]) # fewer than two rounds left+ self.assertEqual(setup._take_run(st, [5, 9, 10]), [5]) # only neighbours go together++ def test_a_broken_run_continues_from_the_byte_it_reached(self):+ cuts = []++ def faults(path, start, end, nth):+ if end - start + 1 > 2 * CHUNK and not cuts:+ cuts.append(start)+ return ("cut", int(2.5 * CHUNK)) # drops inside the third piece of the run+ return None++ with small_pieces(DL_RUN=4, DL_CONNECTIONS=2), Server(faults=faults) as srv, tempfile.TemporaryDirectory() as d:+ code, out = run(srv, d)+ self.assertIsNone(code, out)+ self.assertEqual((Path(d) / NAME).read_bytes(), DATA)+ self.assertEqual(len(cuts), 1)+ first = cuts[0]+ again = [s for s, e in srv.gets if first < s < first + 4 * CHUNK]+ self.assertEqual(len(again), 1) # one more request: for the rest+ self.assertGreaterEqual(again[0], first + 2 * CHUNK) # the two whole pieces are not fetched again++ def test_a_connection_that_keeps_moving_is_not_given_up_on(self):+ """Seven breaks in a row (more than DL_CHUNK_TRIES) that each carried bytes forward do not end the request."""+ count = [0]++ def faults(path, start, end, nth):+ if start < 4 * CHUNK and end - start + 1 > CHUNK // 2 and count[0] < 7:+ count[0] += 1+ return ("cut", CHUNK // 3)+ return None++ with small_pieces(DL_RUN=4, DL_CONNECTIONS=2, DL_ROUNDS=1), Server(faults=faults) as srv, tempfile.TemporaryDirectory() as d:+ code, out = run(srv, d)+ self.assertIsNone(code, out)+ self.assertEqual((Path(d) / NAME).read_bytes(), DATA)+ self.assertEqual(count[0], 7)++ def test_a_request_without_progress_still_gives_up(self):+ def faults(path, start, end, nth):+ if start >= 4 * CHUNK:+ return ("status", 500)+ return None++ with small_pieces(DL_RUN=4, DL_ROUNDS=1, DL_CHUNK_TRIES=2), Server(faults=faults) as srv, tempfile.TemporaryDirectory() as d:+ code, out = run(srv, d)+ self.assertEqual(code, 1)+ self.assertIn("could not finish downloading", out)++ def test_the_default_is_many_connections_and_runs(self):+ self.assertGreaterEqual(setup.DL_CONNECTIONS, 8)+ self.assertGreaterEqual(setup.DL_RUN, 2)+ self.assertEqual(setup.DL_CHUNK, 32 << 20) # the piece list on disk names this size: resumes need it