Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Downloads: 16 connections, several pieces per request, ModelScope length probe, faster MTP fetch

Niko1221e2071c2f8b12
diff --git a/setup.py b/setup.py
index 328aa5b1f..c7a2d32aa 100644
--- a/setup.py
+++ b/setup.py
@@ -1913,12 +1913,14 @@ def drop_archive(z: Path) -> None:
# each connection): download() takes the file in DL_CHUNK pieces over several connections, writes each piece at its own offset,
# and keeps the finished pieces in <name>.part.chunks so a stopped download continues where it was. Anything that is not a
# ranged file of known length (no Range support, no length, a small file) is one stream, as before.
-DL_CONNECTIONS = 8 # STRATA_DL_CONNECTIONS=1 restores one stream (for testers)
-DL_CHUNK = 32 << 20 # bytes per ranged request; a failed request loses at most this much
+DL_CONNECTIONS = 16 # STRATA_DL_CONNECTIONS=1 restores one stream (for testers)
+DL_CHUNK = 32 << 20 # bytes per piece: what the piece list records and what a failed request loses at most
+DL_RUN = 4 # pieces one request asks for (4 x 32 MiB): a new connection per piece spends its first seconds
+ # in TCP slow start (measured: 8 connections 66 MB/s with a request per piece, 73 with 128 MiB)
DL_MIN_PARALLEL = 256 << 20 # smaller files are one stream
-DL_CHUNK_TRIES = 5 # tries per piece (1, 2, 4, 8 s apart) before the round is given up and run again
+DL_CHUNK_TRIES = 5 # tries per request (1, 2, 4, 8 s apart) before the round is given up and run again
DL_ROUNDS = 30 # rounds (10 s apart) before the download is given up, like the 30 tries of one stream
-DL_BACKOFF = 1.0 # seconds before the second try of a piece, doubled each time (tests set 0)
+DL_BACKOFF = 1.0 # seconds before the second try of a request, doubled each time (tests set 0)
DL_SAVE_EVERY = 5.0 # seconds between writes of the progress file
@@ -1994,8 +1996,8 @@ class _Source:
class _Pieces:
"""The shared state of one parallel download: finished pieces, bytes of the pieces in flight, the stop flags."""
- def __init__(self, total: int, done: set):
- self.total, self.n = total, -(-total // DL_CHUNK)
+ def __init__(self, total: int, done: set, conns: int = 1):
+ self.total, self.n, self.conns = total, -(-total // DL_CHUNK), conns
self.done, self.flight = set(done), {}
self.lock = threading.Lock()
self.stop = threading.Event() # the whole download stops (Ctrl-C, a fatal error)
@@ -2104,16 +2106,32 @@ def _single_ready(part: Path):
side.unlink(missing_ok=True)
-def _fetch_piece(src: _Source, st: _Pieces, f, i: int):
- start = i * DL_CHUNK
- end = start + st.size(i) - 1
+def _take_run(st: _Pieces, todo: list) -> list:
+ """The next request's pieces: up to DL_RUN neighbours from the front of `todo`; fewer near the end, so the last pieces
+ are spread over every connection and no single slow one holds the finish."""
+ with st.lock:
+ if not todo:
+ return []
+ n = max(1, min(DL_RUN, len(todo) // (2 * st.conns)))
+ run = [todo.pop(0)]
+ while len(run) < n and todo and todo[0] == run[-1] + 1:
+ run.append(todo.pop(0))
+ return run
+
+
+def _fetch_run(src: _Source, st: _Pieces, f, run: list):
+ """One ranged request for the neighbouring pieces in `run`, written at their offsets; each piece counts as finished
+ when its last byte is down. A connection that breaks continues from the byte it reached, not from the piece start."""
+ start = run[0] * DL_CHUNK
+ end = min((run[-1] + 1) * DL_CHUNK, st.total) - 1
pos, tries = start, 0
with st.lock:
- st.flight[i] = 0
+ for i in run:
+ st.flight[i] = 0
while True:
if st.stop.is_set():
return
- url, err, wait = None, None, DL_BACKOFF * 2 ** tries
+ url, err, got, wait, ra = None, None, False, None, 0.0
try:
url = src.get()
req = urllib.request.Request(url, headers={"User-Agent": "strata-setup", "Range": f"bytes={pos}-{end}"})
@@ -2123,7 +2141,9 @@ def _fetch_piece(src: _Source, st: _Pieces, f, i: int):
while pos <= end:
if st.stop.is_set():
return
- b = r.read(min(1 << 20, end + 1 - pos))
+ cur = pos // DL_CHUNK
+ cur_end = min((cur + 1) * DL_CHUNK, st.total) # one past the piece's last byte
+ b = r.read(min(1 << 20, cur_end - pos))
if not b:
raise IOError("the connection closed early")
try:
@@ -2132,8 +2152,16 @@ def _fetch_piece(src: _Source, st: _Pieces, f, i: int):
except OSError as e:
raise _DiskError(str(e)) from e
pos += len(b)
- st.add(i, len(b))
- st.finish(i)
+ got = True
+ st.add(cur, len(b))
+ if pos == cur_end:
+ st.finish(cur)
+ if pos <= end and st.abort_round.is_set(): # a piece failed for good: stop at a piece edge
+ with st.lock:
+ for i in run:
+ if i not in st.done and st.flight.get(i) == 0:
+ st.flight.pop(i, None)
+ return
return
except (_NoRange, _DiskError):
raise
@@ -2149,26 +2177,25 @@ def _fetch_piece(src: _Source, st: _Pieces, f, i: int):
pass
elif e.code in (429, 503):
try:
- wait = max(wait, min(float(e.headers.get("Retry-After") or 0), 60))
+ ra = min(float(e.headers.get("Retry-After") or 0), 60)
except (TypeError, ValueError):
pass
except (OSError, http.client.HTTPException) as e:
err = e
- tries += 1
+ tries = 1 if got else tries + 1 # a try that moved forward is not a failed one
if tries >= DL_CHUNK_TRIES:
raise _PieceFailed(str(err))
- st.stop.wait(wait)
+ st.stop.wait(max(DL_BACKOFF * 2 ** (tries - 1), ra) if wait is None else wait)
def _piece_worker(src: _Source, st: _Pieces, todo: list, part: Path):
try:
with open(part, "r+b", buffering=0) as f:
while not (st.stop.is_set() or st.abort_round.is_set()):
- with st.lock:
- if not todo:
- return
- i = todo.pop(0)
- _fetch_piece(src, st, f, i)
+ run = _take_run(st, todo)
+ if not run:
+ return
+ _fetch_run(src, st, f, run)
except _PieceFailed as e:
st.errors.append(e)
st.abort_round.set()
@@ -2191,7 +2218,7 @@ def parallel_download(url: str, part: Path, total: int, label: str, conns: int,
except (OSError, http.client.HTTPException):
return False # the one-stream path has its own retries and messages
done = _load_pieces(part, total)
- st = _Pieces(total, done)
+ st = _Pieces(total, done, conns)
part.parent.mkdir(parents=True, exist_ok=True)
if not part.exists():
part.touch()
@@ -2292,6 +2319,18 @@ def download_single(url: str, part: Path, total: int, label: str):
time.sleep(10)
+def range_total(url: str) -> int:
+ """The file's length from a one-byte ranged GET (Content-Range: bytes 0-0/N), for a server whose HEAD gives none
+ (ModelScope's: #1915); 0 when it does not say. With the length known the file comes over several connections."""
+ try:
+ req = urllib.request.Request(url, headers={"User-Agent": "strata-setup", "Range": "bytes=0-0"})
+ with _urlopen(req, timeout=60) as r:
+ m = re.match(r"\s*bytes 0-0/(\d+)\s*$", r.headers.get("Content-Range") or "")
+ return int(m.group(1)) if r.status == 206 and m else 0
+ except (OSError, http.client.HTTPException):
+ return 0
+
+
def download(url, dst: Path, what=None):
"""Resumable HTTP(S) download with a progress line; `file://` and plain paths are copied (tests, mirrors).
A finished file gets a <name>.done mark, so a later run skips it without asking the server."""
@@ -2343,6 +2382,8 @@ def download(url, dst: Path, what=None):
fail(f"cannot reach {url.split('/')[2]} ({e})",
"check your internet connection and run it again" + source_hint(url))
time.sleep(5)
+ if not total:
+ total = range_total(url)
if pin and total and total != pin[0]: # not the pinned file: stop before the download, not after
fail(f"{what or dst.name} is {total:,} bytes on {url.split('/')[2]}, not the pinned {pin[0]:,}",
"update Strata (the file there has changed since this version pinned it)")
diff --git a/tools/mtp_fetch.py b/tools/mtp_fetch.py
index 04d8f8b88..b4a44dfd5 100644
--- a/tools/mtp_fetch.py
+++ b/tools/mtp_fetch.py
@@ -24,6 +24,7 @@ import json
import os
import struct
import sys
+import threading
import time
import urllib.error
import urllib.request
@@ -115,20 +116,41 @@ SHA256 = {
}
+# Hugging Face answers a /resolve/ address with a redirect to a signed CDN address (valid for about an hour). Every range of a
+# tensor used to go through that redirect and a fresh connection to it; the CDN address of the first answer is reused now
+# (and forgotten after any error, so the next try asks again).
+CDN_TTL = 600
+_cdn = {}
+_cdn_lock = threading.Lock()
+
+
+def _final(url):
+ with _cdn_lock:
+ hit = _cdn.get(url)
+ return hit[0] if hit and hit[1] > time.monotonic() else url
+
+
def get(url, start=None, end=None, retries=4):
for attempt in range(retries):
try:
- req = urllib.request.Request(url, headers={"User-Agent": "strata-mtp-fetch"})
+ used = _final(url)
+ req = urllib.request.Request(used, headers={"User-Agent": "strata-mtp-fetch"})
if start is not None:
req.add_header("Range", "bytes=%d-%d" % (start, end))
with urllib.request.urlopen(req, timeout=120) as r:
if start is not None:
check_range(r.status, r.headers.get("Content-Range"), start, end)
data = r.read()
+ landed = getattr(r, "geturl", lambda: url)()
if start is not None and len(data) != end - start + 1:
raise IOError("short range read: %d of %d" % (len(data), end - start + 1))
+ if used == url and landed and landed != url:
+ with _cdn_lock:
+ _cdn[url] = (landed, time.monotonic() + CDN_TTL)
return data
except Exception as e: # network errors are retried, then surfaced
+ with _cdn_lock:
+ _cdn.pop(url, None)
if attempt == retries - 1:
raise
time.sleep(2 ** attempt)
@@ -136,11 +158,11 @@ def get(url, start=None, end=None, retries=4):
def connections():
- """Ranges read at once: STRATA_DL_CONNECTIONS (the setting of setup's downloads), else 8; 1 reads them one by one."""
+ """Ranges read at once: STRATA_DL_CONNECTIONS (the setting of setup's downloads), else 16; 1 reads them one by one."""
try:
- n = int((os.environ.get("STRATA_DL_CONNECTIONS") or "").strip() or 8)
+ n = int((os.environ.get("STRATA_DL_CONNECTIONS") or "").strip() or 16)
except ValueError:
- n = 8
+ n = 16
return max(1, min(n, 32))
@@ -158,10 +180,13 @@ def ranged(url, start, last, chunk, workers):
return
todo = iter(spans)
window = collections.deque()
+ # the ranges are handed on in order, so one slow range holds the oldest slot; the connections that are done go on with
+ # the next ranges meanwhile (up to `depth` ahead, at most ~512 MiB in memory) instead of waiting for it
+ depth = max(workers, min(3 * workers, (512 << 20) // max(chunk, 1)))
with concurrent.futures.ThreadPoolExecutor(min(workers, len(spans))) as ex:
for a, b in todo:
window.append(ex.submit(get, url, a, b))
- if len(window) >= workers:
+ if len(window) >= depth:
break
while window:
data = window.popleft().result()
@@ -225,8 +250,11 @@ def inventory(out):
index = json.loads(get(REPO + "model.safetensors.index.json"))["weight_map"]
mtp = {k: v for k, v in index.items() if k.startswith("mtp.")}
rows, total = [], 0
- for shard in sorted(set(mtp.values())):
- base, header = shard_header(shard)
+ shards = sorted(set(mtp.values()))
+ # one small request pair per shard (28 of them): side by side, not one after the other (28 s -> 4 s)
+ with concurrent.futures.ThreadPoolExecutor(max(1, min(connections(), 8, len(shards)))) as ex:
+ headers = list(ex.map(shard_header, shards))
+ for shard, (base, header) in zip(shards, headers):
for name, meta in header.items():
if name in mtp and mtp[name] == shard:
a, b = meta["data_offsets"]
@@ -268,7 +296,7 @@ def fetch(out, only):
os.makedirs(tdir, exist_ok=True)
manifest = []
workers = connections()
- chunk = (32 if workers > 1 else 64) << 20
+ chunk = (16 if workers > 1 else 64) << 20
for r in rows:
if only and only not in r["name"]:
continue
diff --git a/tools/test_mtp_fetch.py b/tools/test_mtp_fetch.py
index cb66527c7..bf2476506 100644
--- a/tools/test_mtp_fetch.py
+++ b/tools/test_mtp_fetch.py
@@ -145,7 +145,60 @@ class Ranges(FetchCase):
with mock.patch.dict(M.os.environ, {"STRATA_DL_CONNECTIONS": "1"}):
self.assertEqual(M.connections(), 1)
with mock.patch.dict(M.os.environ, {"STRATA_DL_CONNECTIONS": ""}):
- self.assertEqual(M.connections(), 8)
+ self.assertEqual(M.connections(), 16)
+
+ def test_the_cdn_address_is_reused_and_forgotten_after_an_error(self):
+ """HF redirects /resolve/ to a signed CDN address: only the first range goes through the redirect (the rest of a
+ tensor's ranges use the CDN address), and any error makes the next try ask the original address again."""
+ asked = []
+
+ def opener(req, timeout=None):
+ asked.append(req.full_url)
+ a, b = (int(x) for x in req.headers["Range"].split("=")[1].split("-"))
+ r = Response(bytes(b - a + 1), 206, {"Content-Range": "bytes %d-%d/1000" % (a, b)})
+ r.geturl = lambda: "https://cdn.example/x?sig=1"
+ return r
+
+ M._cdn.clear()
+ with mock.patch.object(M.urllib.request, "urlopen", opener):
+ for a in (0, 100, 200):
+ M.get("https://hf.example/f", a, a + 99)
+ self.assertEqual(asked, ["https://hf.example/f"] + ["https://cdn.example/x?sig=1"] * 2)
+ asked.clear()
+ fails = [True]
+
+ def flaky(req, timeout=None):
+ if fails.pop() if fails else False:
+ raise IOError("expired")
+ return opener(req, timeout)
+
+ with mock.patch.object(M.urllib.request, "urlopen", flaky), mock.patch.object(M.time, "sleep", lambda s: None), contextlib.redirect_stderr(io.StringIO()):
+ M.get("https://hf.example/f", 300, 399)
+ self.assertEqual(asked, ["https://hf.example/f"]) # the cached address failed: the original is asked again
+ M._cdn.clear()
+
+ def test_a_slow_first_range_does_not_stop_the_ones_behind_it(self):
+ import threading
+ gate = threading.Event()
+ started = []
+
+ def get(url, a, b):
+ started.append(a)
+ if a == 0:
+ gate.wait(5)
+ return bytes(b - a + 1)
+
+ def release(): # lets the first range go once the later ones have been started
+ for _ in range(200):
+ if len(started) >= 9:
+ break
+ M.time.sleep(0.01)
+ gate.set()
+
+ threading.Thread(target=release, daemon=True).start()
+ with mock.patch.object(M, "get", get):
+ list(M.ranged("u", 0, 40 * 1000 - 1, 1000, 4))
+ self.assertGreaterEqual(len(started), 9) # four connections, nine ranges started before range 0 ended
def test_a_failing_range_stops_the_parallel_read(self):
def get(url, a, b):
diff --git a/tools/test_setup_download.py b/tools/test_setup_download.py
index a14f14056..fc2e3236c 100644
--- a/tools/test_setup_download.py
+++ b/tools/test_setup_download.py
@@ -142,7 +142,7 @@ def pin_of(data):
@contextlib.contextmanager
def small_pieces(**extra):
"""Setup's piece size, size floor and waits scaled down to the test file."""
- consts = dict(DL_CHUNK=CHUNK, DL_MIN_PARALLEL=0, DL_BACKOFF=0.0, DL_SAVE_EVERY=0.05, DL_CONNECTIONS=4)
+ consts = dict(DL_CHUNK=CHUNK, DL_RUN=1, DL_MIN_PARALLEL=0, DL_BACKOFF=0.0, DL_SAVE_EVERY=0.05, DL_CONNECTIONS=4)
consts.update(extra)
with contextlib.ExitStack() as st:
for k, v in consts.items():
@@ -230,28 +230,38 @@ class NoRangeSupport(unittest.TestCase):
self.assertEqual(srv.max_active, 1)
self.assertEqual(files(d), [NAME, NAME + ".done"])
- def test_a_server_without_a_length_is_read_in_one_stream(self):
- with small_pieces(), Server(length=False) as srv, tempfile.TemporaryDirectory() as d:
+ def test_a_server_without_any_length_is_read_in_one_stream(self):
+ with small_pieces(), Server(length=False, ranges=False) as srv, tempfile.TemporaryDirectory() as d:
code, out = run(srv, d, pin=None) # no length, so nothing to check a pin against first
self.assertIsNone(code, out)
self.assertEqual((Path(d) / NAME).read_bytes(), DATA)
self.assertEqual(srv.max_active, 1)
+ def test_a_head_without_a_length_still_gets_the_connections(self):
+ """ModelScope's HEAD has no Content-Length (#1915): the length comes from a one-byte ranged GET, not 'unknown'."""
+ with small_pieces(), Server(length=False) as srv, tempfile.TemporaryDirectory() as d:
+ code, out = run(srv, d)
+ self.assertIsNone(code, out)
+ self.assertEqual((Path(d) / NAME).read_bytes(), DATA)
+ self.assertGreater(srv.max_active, 1)
+ self.assertEqual(sorted(asked(srv)), list(range(41)))
+ self.assertEqual(files(d), [NAME, NAME + ".done"])
+
def test_range_stopping_in_the_middle_falls_back_to_the_unbroken_start(self):
def faults(path, start, end, nth):
return None
with small_pieces(), Server(faults=faults) as srv, tempfile.TemporaryDirectory() as d:
- real = setup._fetch_piece
+ real = setup._fetch_run
calls = []
- def fetch(src, st, f, i):
- calls.append(i)
+ def fetch(src, st, f, run):
+ calls.append(run)
if len(calls) == 6:
raise setup._NoRange("HTTP 200")
- return real(src, st, f, i)
+ return real(src, st, f, run)
- with mock.patch.object(setup, "_fetch_piece", fetch):
+ with mock.patch.object(setup, "_fetch_run", fetch):
code, out = run(srv, d)
self.assertIsNone(code, out)
self.assertEqual((Path(d) / NAME).read_bytes(), DATA) # the pin check passed: no gaps were kept
@@ -471,3 +481,80 @@ class SignedAddress(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
+
+class Runs(unittest.TestCase):
+ """One request asks for several neighbouring pieces (DL_RUN): fewer connections to open, the same piece list."""
+
+ def test_a_request_covers_several_pieces(self):
+ with small_pieces(DL_RUN=4), Server() as srv, tempfile.TemporaryDirectory() as d:
+ code, out = run(srv, d)
+ self.assertIsNone(code, out)
+ self.assertEqual((Path(d) / NAME).read_bytes(), DATA)
+ self.assertEqual(files(d), [NAME, NAME + ".done"])
+ spans = [(s, e) for s, e in srv.gets if (s, e) != (0, 0)]
+ self.assertLess(len(spans), 41) # fewer requests than pieces
+ self.assertTrue(any(e - s + 1 > CHUNK for s, e in spans), spans)
+ self.assertTrue(all(e - s + 1 <= 4 * CHUNK for s, e in spans), spans)
+ covered = sorted(i for s, e in spans for i in range(s // CHUNK, e // CHUNK + 1))
+ self.assertEqual(covered, list(range(41))) # every piece once, none twice
+
+ def test_the_last_pieces_are_asked_for_one_by_one(self):
+ st = setup._Pieces(CHUNK * 100, set(), conns=4)
+ todo = list(range(100))
+ self.assertEqual(setup._take_run(st, todo), [0, 1, 2, 3])
+ with mock.patch.object(setup, "DL_RUN", 4):
+ self.assertEqual(setup._take_run(st, list(range(60, 100))), [60, 61, 62, 63])
+ self.assertEqual(setup._take_run(st, list(range(93, 100))), [93]) # fewer than two rounds left
+ self.assertEqual(setup._take_run(st, [5, 9, 10]), [5]) # only neighbours go together
+
+ def test_a_broken_run_continues_from_the_byte_it_reached(self):
+ cuts = []
+
+ def faults(path, start, end, nth):
+ if end - start + 1 > 2 * CHUNK and not cuts:
+ cuts.append(start)
+ return ("cut", int(2.5 * CHUNK)) # drops inside the third piece of the run
+ return None
+
+ with small_pieces(DL_RUN=4, DL_CONNECTIONS=2), Server(faults=faults) as srv, tempfile.TemporaryDirectory() as d:
+ code, out = run(srv, d)
+ self.assertIsNone(code, out)
+ self.assertEqual((Path(d) / NAME).read_bytes(), DATA)
+ self.assertEqual(len(cuts), 1)
+ first = cuts[0]
+ again = [s for s, e in srv.gets if first < s < first + 4 * CHUNK]
+ self.assertEqual(len(again), 1) # one more request: for the rest
+ self.assertGreaterEqual(again[0], first + 2 * CHUNK) # the two whole pieces are not fetched again
+
+ def test_a_connection_that_keeps_moving_is_not_given_up_on(self):
+ """Seven breaks in a row (more than DL_CHUNK_TRIES) that each carried bytes forward do not end the request."""
+ count = [0]
+
+ def faults(path, start, end, nth):
+ if start < 4 * CHUNK and end - start + 1 > CHUNK // 2 and count[0] < 7:
+ count[0] += 1
+ return ("cut", CHUNK // 3)
+ return None
+
+ with small_pieces(DL_RUN=4, DL_CONNECTIONS=2, DL_ROUNDS=1), Server(faults=faults) as srv, tempfile.TemporaryDirectory() as d:
+ code, out = run(srv, d)
+ self.assertIsNone(code, out)
+ self.assertEqual((Path(d) / NAME).read_bytes(), DATA)
+ self.assertEqual(count[0], 7)
+
+ def test_a_request_without_progress_still_gives_up(self):
+ def faults(path, start, end, nth):
+ if start >= 4 * CHUNK:
+ return ("status", 500)
+ return None
+
+ with small_pieces(DL_RUN=4, DL_ROUNDS=1, DL_CHUNK_TRIES=2), Server(faults=faults) as srv, tempfile.TemporaryDirectory() as d:
+ code, out = run(srv, d)
+ self.assertEqual(code, 1)
+ self.assertIn("could not finish downloading", out)
+
+ def test_the_default_is_many_connections_and_runs(self):
+ self.assertGreaterEqual(setup.DL_CONNECTIONS, 8)
+ self.assertGreaterEqual(setup.DL_RUN, 2)
+ self.assertEqual(setup.DL_CHUNK, 32 << 20) # the piece list on disk names this size: resumes need it