Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: vector cache in float32 unit rows with streamed builds, in-place appends and bounded block scans (#16093)
Daniel Han6fe27d2c5cd6
diff --git a/studio/backend/core/rag/vec_cache.py b/studio/backend/core/rag/vec_cache.pyindex fecf8da8147..d649b74794e 100644--- a/studio/backend/core/rag/vec_cache.py+++ b/studio/backend/core/rag/vec_cache.py@@ -41,13 +41,67 @@ def _budget() -> int:return 0+# Rows per matvec: per-query scratch stays O(block) however large the scope.+_BLOCK = 32768+_FETCH = 4096+++class _Store:+ """Growable unit-row float32 matrix + rowids + chunk ids. Successive entries of one scope share it: an+ entry reads only its first ``n`` rows and appends write past every published ``n``."""++ __slots__ = ("mat", "rowids", "ids")++ def __init__(self, np, cap: int, dim: int):+ cap = max(int(cap), 1)+ self.mat = np.empty((cap, dim), dtype = np.float32)+ self.rowids = np.empty(cap, dtype = np.int64)+ self.ids = np.empty(cap, dtype = object)++ def grown(self, np, n: int, need: int) -> "_Store":+ """Self if ``need`` rows fit, else a copy of the first ``n`` rows with geometric headroom."""+ cap = len(self.rowids)+ if need <= cap:+ return self+ new = _Store(np, max(need, cap + cap // 4 + 16), self.mat.shape[1])+ new.mat[:n] = self.mat[:n]+ new.rowids[:n] = self.rowids[:n]+ new.ids[:n] = self.ids[:n]+ return new++ def write(self, np, n: int, rows, rowids, ids) -> int:+ """Normalise ``rows`` (float32, writable) in place and store the scorable ones at ``n``; returns the new n."""+ valid = _normalise(np, rows)+ keep = np.flatnonzero(valid)+ k = len(keep)+ if k == len(rows):+ self.mat[n : n + k] = rows+ self.rowids[n : n + k] = rowids+ self.ids[n : n + k] = ids+ else:+ self.mat[n : n + k] = rows[keep]+ self.rowids[n : n + k] = np.asarray(rowids, dtype = np.int64)[keep]+ self.ids[n : n + k] = [ids[i] for i in keep]+ return n + k++ def nbytes(self, avg_id: int) -> int:+ return self.mat.nbytes + self.rowids.nbytes + len(self.rowids) * (avg_id + _ID_OVERHEAD)+++def _normalise(np, rows):+ """Unit rows in place; mask of the rows vec0 can score (it answers NULL for NaN / inf / zero rows)."""+ with np.errstate(all = "ignore"):+ norms = np.sqrt(np.einsum("ij,ij->i", rows, rows))+ valid = np.isfinite(norms) & (norms > 0)+ rows /= norms[:, None]+ return valid++@dataclass(frozen = True)class _Entry:epoch: tuple- rowids: object- chunk_ids: object- mat: object- inv: object+ n: int+ store: _Storenbytes: int@@ -84,24 +138,6 @@ def read_epoch(conn: sqlite3.Connection, scope: str) -> tuple | None:return None-def _make_entry(np, epoch, rowids, chunk_ids, blob: bytes, dim: int) -> _Entry:- mat = np.frombuffer(blob, dtype = np.float32).reshape(-1, dim)- with np.errstate(all = "ignore"):- norms = np.sqrt(np.einsum("ij,ij->i", mat, mat, dtype = np.float64))- # vec0 answers NULL for NaN / inf / zero rows, which _search_dense drops.- valid = np.isfinite(norms) & (norms > 0)- mat = np.ascontiguousarray(mat[valid])- inv = (1.0 / norms[valid]).astype(np.float32)- rowids = np.asarray(rowids, dtype = np.int64)[valid]- ids = np.empty(len(chunk_ids), dtype = object)- ids[:] = chunk_ids- ids = ids[valid]- for a in (mat, inv, rowids, ids):- a.setflags(write = False)- nbytes = mat.nbytes + inv.nbytes + rowids.nbytes + sum(len(c) + _ID_OVERHEAD for c in ids)- return _Entry(epoch, rowids, ids, mat, inv, nbytes)--def _publish(key: tuple,entry: _Entry | None,@@ -155,18 +191,43 @@ def search(conn, db_file: str, scope: str, vector, fetch: int):return None+def _top(np, scores, f: int):+ """Indices of the ``f`` largest scores, unordered."""+ m = len(scores)+ return np.argpartition(scores, m - f)[m - f :] if f < m else np.arange(m)++def _search_entry(np, entry: _Entry, vector, fetch: int):- q = np.asarray(vector, dtype = np.float32)- qn = float(np.sqrt(np.dot(q.astype(np.float64), q.astype(np.float64))))- n = len(entry.rowids)+ n = entry.n+ q = np.array(vector, dtype = np.float32)+ qn = float(np.sqrt(np.dot(q, q)))if not np.isfinite(qn) or qn == 0.0 or n == 0:return [], n- scores = (entry.mat @ q) * entry.inv- scores = scores.astype(np.float64) / qn+ q /= qn+ store = entry.storef = min(int(fetch), n)- idx = np.argpartition(-scores, f - 1)[:f] if f < n else np.arange(n)- order = idx[np.lexsort((entry.rowids[idx], -scores[idx]))]- return [(entry.chunk_ids[i], 1.0 - float(scores[i])) for i in order], n+ if n <= _BLOCK:+ scores = store.mat[:n] @ q+ idx = _top(np, scores, f)+ best = scores[idx]+ else:+ buf = np.empty(_BLOCK, dtype = np.float32)+ idx = np.empty(0, dtype = np.int64)+ best = np.empty(0, dtype = np.float32)+ # Running top-f: scratch is O(block + fetch), not O(scope).+ for s0 in range(0, n, _BLOCK):+ out = buf[: min(n, s0 + _BLOCK) - s0]+ np.matmul(store.mat[s0 : s0 + len(out)], q, out = out)+ part = _top(np, out, min(f, len(out)))+ idx = np.concatenate([idx, part + s0])+ best = np.concatenate([best, out[part]])+ if len(best) > f:+ keep = _top(np, best, f)+ idx, best = idx[keep], best[keep]+ # Same order vec0 gives: distance ascending, ties by rowid.+ order = np.lexsort((store.rowids[idx], -best))+ ids = store.ids+ return [(ids[i], 1.0 - float(s)) for i, s in zip(idx[order], best[order])], ndef note_append(@@ -182,6 +243,7 @@ def note_append(try:dim = len(vectors[0])key = _key(db_file, scope, dim)+ rows = np.array(vectors, dtype = np.float32).reshape(len(rowids), dim)with _lock:entry = _entries.get(key)if entry is None:@@ -189,20 +251,10 @@ def note_append(if after[:2] != before[:2] or appended != len(rowids) or entry.epoch != before:_publish(key, None)return- rows = np.asarray(vectors, dtype = np.float32).reshape(len(rowids), dim)- add = _make_entry(np, after, rowids, list(chunk_ids), rows.tobytes(), dim)- merged = _Entry(- after,- np.concatenate([entry.rowids, add.rowids]),- np.concatenate([entry.chunk_ids, add.chunk_ids]),- np.concatenate([entry.mat, add.mat]),- np.concatenate([entry.inv, add.inv]),- entry.nbytes + add.nbytes,- )- for a in (merged.rowids, merged.chunk_ids, merged.mat, merged.inv):- a.setflags(write = False)- with _lock:- _publish(key, merged, expect = before)+ # Under the lock with the epoch check, so no other append writes the same rows.+ store = entry.store.grown(np, entry.n, entry.n + len(rows))+ n = store.write(np, entry.n, rows, rowids, list(chunk_ids))+ _publish(key, _Entry(after, n, store, store.nbytes(_avg_id(store, n))), expect = before)except Exception: # noqa: BLE001logger.debug("vec_cache.note_append failed", exc_info = True)with _lock:@@ -268,28 +320,34 @@ def _build(key: tuple) -> None:if epoch is None or rag_db.vec_table_dim(conn) != dim:returncount = conn.execute("SELECT count(*) FROM chunks WHERE scope=?", (scope,)).fetchone()[0]- if count * (dim * 4 + _ID_OVERHEAD) > _budget():+ if count * (dim * 4 + 8 + _ID_OVERHEAD) > _budget():with _lock:_too_big[key] = epochreturn- rows = conn.execute(+ # Streamed into one preallocated matrix: peak is the matrix plus one fetch block.+ store = _Store(np, count, dim)+ n = 0+ cur = conn.execute("SELECT rowid, chunk_id, embedding FROM chunks_vec WHERE scope=?", (scope,)- ).fetchall()+ )+ while rows := cur.fetchmany(_FETCH):+ blk = np.frombuffer(b"".join(r[2] for r in rows), dtype = np.float32)+ blk = blk.reshape(len(rows), dim).copy()+ store = store.grown(np, n, n + len(rows))+ n = store.write(np, n, blk, [r[0] for r in rows], [r[1] for r in rows])finally:conn.close()- entry = _make_entry(- np,- epoch,- [r[0] for r in rows],- [r[1] for r in rows],- b"".join(r[2] for r in rows),- dim,- )+ entry = _Entry(epoch, n, store, store.nbytes(_avg_id(store, n)))with _lock:_too_big.pop(key, None)_publish(key, entry)+def _avg_id(store: _Store, n: int) -> int:+ sample = store.ids[: min(n, 64)]+ return int(sum(len(c) for c in sample) / len(sample)) if len(sample) else 0++def drain(timeout: float = 30.0) -> bool:"""Wait for queued builds; for tests and benchmarks."""with _lock:diff --git a/studio/backend/tests/test_rag_vec_cache.py b/studio/backend/tests/test_rag_vec_cache.pyindex 8c7ed31ac93..4629c8ab1d6 100644--- a/studio/backend/tests/test_rag_vec_cache.py+++ b/studio/backend/tests/test_rag_vec_cache.py@@ -87,7 +87,14 @@ def _hit(conn, scope, q, k, monkeypatch):return got-def test_cache_matches_vec0_on_unnormalised_nan_zero_and_tied_rows(rag_conn, monkeypatch):[email protected]("block, fetch", [(None, None), (7, 11)])+def test_cache_matches_vec0_on_unnormalised_nan_zero_and_tied_rows(+ rag_conn, monkeypatch, block, fetch+):+ # Small sizes push the build across fetch blocks and the scan across matvec blocks.+ if block:+ monkeypatch.setattr(vec_cache, "_BLOCK", block)+ monkeypatch.setattr(vec_cache, "_FETCH", fetch)rows = _vectors(300)rows[5] = np.nanrows[9] = 0.0@@ -291,3 +298,46 @@ def test_the_builder_opens_and_closes_through_the_connection_gate(monkeypatch, tconn = vec_cache._connect(str(tmp_path / "x.db"))assert isinstance(conn, sqlite_gate.GatedConnection) and len(opened) == 1conn.close()+++def test_appends_write_in_place_and_an_older_entry_keeps_its_rows(rag_conn, monkeypatch):+ rows = _vectors(40)+ _add(rag_conn, "kb_a", "d", rows)+ q = rows[3]+ _warm(rag_conn, "kb_a", q, monkeypatch)+ (old,) = vec_cache._entries.values()+ before = _hit(rag_conn, "kb_a", q, 10, monkeypatch)+ stores = {id(old.store)}+ for i in range(6):+ _add(rag_conn, "kb_a", f"e{i}", _vectors(5, seed = 100 + i))+ (cur,) = vec_cache._entries.values()+ stores.add(id(cur.store))+ assert cur.n == 70+ # Geometric growth: six appends reallocate at most a couple of times, not once each.+ assert len(stores) <= 3+ # _search_entry gives distances; search_dense gives 1 - distance.+ assert vec_cache._search_entry(np, old, q, 10)[0] == [+ (c, pytest.approx(1 - s, abs = 1e-6)) for c, s in before+ ]+ _same(+ _hit(rag_conn, "kb_a", q, 70, monkeypatch),+ _search(rag_conn, "kb_a", q, 70, monkeypatch, False),+ )+++def test_block_scan_scratch_stays_bounded_by_block_plus_fetch(monkeypatch):+ import tracemalloc++ monkeypatch.setattr(vec_cache, "_BLOCK", 4096)+ n, f = 200_000, 4096+ store = vec_cache._Store(np, n, 1)+ store.mat[:] = np.random.default_rng(0).standard_normal((n, 1), dtype = np.float32)+ store.rowids[:] = np.arange(n)+ store.ids[:] = "c"+ entry = vec_cache._Entry((0, 0, 0), n, store, 0)+ tracemalloc.start()+ vec_cache._search_entry(np, entry, [1.0], f)+ peak = tracemalloc.get_traced_memory()[1]+ tracemalloc.stop()+ # Result list aside, a scan that kept every block's candidates would hold ~n * 12 bytes (2.4 MB).+ assert peak < 1.5 * 2**20, peak