Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fit knowledge base chunks to the embedding model's input limit (#14045)
Nilay0b75841dae1b
diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.pyindex 484644d8023..330409f1e6b 100644--- a/studio/backend/core/rag/conversation_archive.py+++ b/studio/backend/core/rag/conversation_archive.py@@ -408,6 +408,7 @@ def archive_turns(conn = Nonetry:count = embeddings.token_counter(model)+ chunk_limit = Noneconn = rag_db.get_connection()# What the query side will ask for. Only a prediction until the encode reports what it actually used, so the# authoritative check happens under the write lock.@@ -441,9 +442,11 @@ def archive_turns()_widen_span(conn, scope, digest, span)continue+ if chunk_limit is None:+ chunk_limit = embeddings.chunk_tokens(model)chunks = chunk_pages([Page(text = text, page_number = None, char_count = len(text))],- max_tokens = config.CHUNK_TOKENS,+ max_tokens = chunk_limit,overlap = config.CHUNK_OVERLAP,count = count,)diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.pyindex 7fc872b96fd..1f36a441df3 100644--- a/studio/backend/core/rag/embeddings.py+++ b/studio/backend/core/rag/embeddings.py@@ -1504,6 +1504,11 @@ def max_tokens(model_name: str | None = None) -> int | None:return _get_backend(model_name).max_tokens(model_name = model_name)+def chunk_tokens(model_name: str | None = None) -> int:+ limit = max_tokens(model_name)+ return min(config.CHUNK_TOKENS, limit) if limit else config.CHUNK_TOKENS++def token_counter(model_name: str | None = None) -> Callable[[str], int]:"""Callable counting tokens with the embedder's own tokenizer.diff --git a/studio/backend/core/rag/ingestion.py b/studio/backend/core/rag/ingestion.pyindex 67bbbce4d17..c8cfcd024cf 100644--- a/studio/backend/core/rag/ingestion.py+++ b/studio/backend/core/rag/ingestion.py@@ -403,7 +403,7 @@ def _run(count = embeddings.token_counter(model_name)chunks = chunking.chunk_pages(pages,- max_tokens = config.CHUNK_TOKENS,+ max_tokens = embeddings.chunk_tokens(model_name),overlap = config.CHUNK_OVERLAP,count = count,)diff --git a/studio/backend/core/rag/web_rank.py b/studio/backend/core/rag/web_rank.pyindex 77cf4a81832..32f89a2a1ac 100644--- a/studio/backend/core/rag/web_rank.py+++ b/studio/backend/core/rag/web_rank.py@@ -70,7 +70,6 @@ def retrieve_web_chunks(if not query or top_n <= 0 or not pages or not rag_db.RAG_AVAILABLE:return "", []model = model_name or config.effective_embedding_model()- max_tokens = max_tokens or config.CHUNK_TOKENSoverlap = config.CHUNK_OVERLAP if overlap is None else overlapcount = embeddings.token_counter(model)@@ -82,6 +81,7 @@ def retrieve_web_chunks(scope = f"research_scrape_{uuid.uuid4().hex}"doc_ids: list[str] = []try:+ max_tokens = max_tokens or embeddings.chunk_tokens(model)for page in pages:text = str(page.get("text") or "").strip()if not text:diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex 5febdc7f7a0..fd900f791e0 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -1110,6 +1110,7 @@ def stub_embeddings(monkeypatch):monkeypatch.setattr(embeddings, "encode", fake_encode)monkeypatch.setattr(embeddings, "dim", lambda model_name = None: dim)+ monkeypatch.setattr(embeddings, "max_tokens", lambda model_name = None: None)monkeypatch.setattr(embeddings,"token_counter",diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.pyindex a16aec7d082..373a17698d3 100644--- a/studio/backend/tests/test_conversation_archive.py+++ b/studio/backend/tests/test_conversation_archive.py@@ -123,6 +123,20 @@ def _tool_part(}+def test_the_embedding_limit_is_read_once_per_pass_and_never_for_archived_turns(conn, monkeypatch):+ from core.rag import embeddings++ calls = []+ monkeypatch.setattr(embeddings, "max_tokens", lambda model_name = None: calls.append(model_name))+ turns = _turn("q1", "a1") + _turn("q2", "a2") + _turn("q3", "a3")+ assert _archive(turns) == 3+ assert len(calls) == 1++ calls.clear()+ assert conversation_archive.archive_turns(THREAD, [dict(m) for m in turns]) == 0+ assert calls == []++def test_evicted_turns_are_archived_under_the_conversation_scope(conn):written = _archive(_turn("what is a duck", "a waterfowl"))diff --git a/studio/backend/tests/test_rag_embedding_identity.py b/studio/backend/tests/test_rag_embedding_identity.pyindex fb2626ca19e..ec4e4454ba8 100644--- a/studio/backend/tests/test_rag_embedding_identity.py+++ b/studio/backend/tests/test_rag_embedding_identity.py@@ -463,6 +463,7 @@ def test_the_web_ranker_labels_a_page_with_the_backend_that_encoded_it(rag_home,monkeypatch.setattr(embeddings, "token_counter", lambda model_name = None: lambda t: max(1, len(t.split())))+ monkeypatch.setattr(embeddings, "max_tokens", lambda model_name = None: None)class _SwapsMidEncode:def encode(diff --git a/studio/backend/tests/test_rag_ingestion.py b/studio/backend/tests/test_rag_ingestion.pyindex 4c7bde2d9a3..30cbea5cf6a 100644--- a/studio/backend/tests/test_rag_ingestion.py+++ b/studio/backend/tests/test_rag_ingestion.py@@ -104,6 +104,26 @@ def test_ingestion_lifecycle_pending_to_completed(rag_home, stub_embeddings, tmpconn.close()+def test_ingested_chunks_fit_the_embedding_model_input_limit(+ rag_home, stub_embeddings, tmp_path, monkeypatch+):+ from core.rag import embeddings++ monkeypatch.setattr(embeddings, "max_tokens", lambda model_name = None: 8)+ path = _write(tmp_path, "doc.txt", " ".join(f"word{i}" for i in range(60)))+ scope = store.kb_scope("K1")+ doc_id, job_id = ingestion.start_ingestion(scope, "K1", None, "doc.txt", path)+ assert _wait_completed(job_id)["status"] == "completed"++ conn = rag_db.get_connection()+ try:+ rows = conn.execute("SELECT text FROM chunks WHERE document_id=?", (doc_id,)).fetchall()+ finally:+ conn.close()+ assert len(rows) > 1+ assert max(len(row["text"].split()) for row in rows) <= 8++def test_ingestion_skips_chunk_write_when_the_document_was_deleted(rag_home, stub_embeddings, tmp_path, monkeypatch):diff --git a/studio/backend/tests/test_web_rank.py b/studio/backend/tests/test_web_rank.pyindex cdb09ab5182..9542b779cf4 100644--- a/studio/backend/tests/test_web_rank.py+++ b/studio/backend/tests/test_web_rank.py@@ -55,6 +55,7 @@ def fake_embeddings(monkeypatch):return np.stack(rows)monkeypatch.setattr(rag_embeddings, "encode", encode)+ monkeypatch.setattr(rag_embeddings, "max_tokens", lambda model_name = None: None)def _scope_rows(db_file):@@ -117,6 +118,15 @@ def test_char_budget_caps_kept_chunks(rag_home):assert 1 <= capped.count("<chunk id") < full.count("<chunk id")+def test_web_chunks_fit_the_embedding_model_input_limit(rag_home, monkeypatch):+ from core.rag import embeddings as rag_embeddings++ monkeypatch.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 8)+ pages = [{"text": " ".join(f"lora{i}" for i in range(60)), "url": "https://a"}]+ rendered, _ = web_rank.retrieve_web_chunks(pages, "lora", top_n = 20, min_score = 0.0)+ assert rendered.count("<chunk id") > 1++def test_empty_and_invalid_inputs_return_empty(rag_home):assert web_rank.retrieve_web_chunks([], "lora", top_n = 5, min_score = 0.1) == ("", [])assert web_rank.retrieve_web_chunks([{"text": " "}], "lora", top_n = 5, min_score = 0.1) == ("", [])