Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Store tests: incremental writes, dedupe, delete, scope, dense + lexical."""
import math
import re
import sqlite3
import pytest
from core.rag import store
from core.rag.chunking import Chunk
from growth import assert_linear # tests/_shared, on sys.path via tests/conftest.py
VOCAB = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel"]
def embed(text):
v = [float(text.lower().count(w)) for w in VOCAB]
n = math.sqrt(sum(x * x for x in v)) or 1.0
return [x / n for x in v]
def _chunk(
text,
index = 0,
page = None,
):
return Chunk(
text = text,
token_count = len(text.split()),
page_number = page,
source_page_index = 0,
chunk_index = index,
page_char_start = 0,
page_char_end = len(text),
)
def _add_doc(conn, scope, doc_id, filename, sha, texts):
chunks = [_chunk(t, i) for i, t in enumerate(texts)]
vectors = [embed(t) for t in texts]
store.create_document(conn, scope = scope, filename = filename, sha256 = sha, document_id = doc_id)
store.add_chunks(conn, scope, doc_id, chunks, vectors)
def test_lexical_returns_only_matching_docs(rag_conn):
_add_doc(rag_conn, "kb_a", "d1", "d1.txt", "h1", ["alpha bravo charlie"])
_add_doc(rag_conn, "kb_a", "d2", "d2.txt", "h2", ["golf hotel india"])
hits = store.search_lexical(rag_conn, "kb_a", "alpha", 10)
assert [cid for cid, _ in hits] == ["d1:0"]
def test_scope_isolation(rag_conn):
_add_doc(rag_conn, "kb_a", "d1", "f", "h1", ["alpha bravo"])
_add_doc(rag_conn, "kb_b", "d2", "f", "h2", ["alpha bravo"])
assert [cid for cid, _ in store.search_lexical(rag_conn, "kb_b", "alpha", 10)] == ["d2:0"]
def test_match_query_sanitizes_special_chars():
assert store._match_query('AND OR "quote" (paren) -dash') != ""
def test_lexical_finds_words_in_unspaced_cjk_text(rag_conn):
_add_doc(rag_conn, "kb_a", "zh", "zh.txt", "h1", ["我们的退款政策:购买后30天内可以退款。"])
_add_doc(rag_conn, "kb_a", "ja", "ja.txt", "h2", ["返金ポリシー:30日以内に返金します。"])
_add_doc(rag_conn, "kb_a", "ko", "ko.txt", "h3", ["환불 정책은 30일 이내입니다."])
_add_doc(rag_conn, "kb_a", "en", "en.txt", "h4", ["Our refund policy: refunds within 30 days."])
def hits(query):
return [cid for cid, _ in store.search_lexical(rag_conn, "kb_a", query, 10)]
assert hits("退款政策是什么?")[0] == "zh:0"
assert hits("退款") == ["zh:0"]
assert hits("返金") == ["ja:0"]
assert hits("ポリシー") == ["ja:0"]
assert hits("환불은 어떻게 되나요") == ["ko:0"]
assert hits("refund policy") == ["en:0"]
assert hits("退") == ["zh:0"]
def test_focused_query_keeps_a_cjk_identifier_whole(rag_conn):
_add_doc(rag_conn, "kb_a", "a", "a.txt", "h1", ["案件123的状态"])
_add_doc(rag_conn, "kb_a", "b", "b.txt", "h2", ["案件124的状态"])
focused = store.conversation_match_queries("current status of 案件123")[0]
assert store.lexical_matching_ids(rag_conn, ["a:0", "b:0"], focused) == {"a:0"}
def test_repeated_cjk_text_builds_each_phrase_once():
terms = store._match_query("订单:苹果手机一台。" * 200).split(" OR ")
assert len(terms) == len(set(terms)) == 6
assert store._match_query("refund refund policy") == '"refund" OR "refund" OR "policy"'
def test_existing_index_is_rebuilt_for_cjk_search(rag_conn):
from storage import rag_db
text = "我们的退款政策:购买后30天内可以退款。"
_add_doc(rag_conn, "kb_a", "zh", "zh.txt", "h1", [text])
_add_doc(rag_conn, "kb_a", "en", "en.txt", "h2", ["alpha bravo"])
rag_conn.execute("UPDATE chunks_fts SET text=? WHERE chunk_id='zh:0'", (text,))
rag_conn.execute("PRAGMA user_version = 0")
rag_conn.commit()
assert store.search_lexical(rag_conn, "kb_a", "退款", 10) == []
rag_db.reset_schema_state_for_tests()
for _ in range(2):
conn = rag_db.get_connection()
try:
assert [cid for cid, _ in store.search_lexical(conn, "kb_a", "退款", 10)] == ["zh:0"]
assert [cid for cid, _ in store.search_lexical(conn, "kb_a", "alpha", 10)] == ["en:0"]
finally:
conn.close()
rag_db.reset_schema_state_for_tests()
assert store.chunks_by_id(rag_conn, ["zh:0"])["zh:0"]["text"] == text
def test_lexical_does_not_crash_on_punctuation(rag_conn):
_add_doc(rag_conn, "kb_a", "d1", "f", "h1", ["alpha bravo"])
store.search_lexical(rag_conn, "kb_a", 'NEAR("x" AND', 5)
def test_dense_ranks_by_cosine(rag_conn):
_add_doc(rag_conn, "kb_a", "d1", "f", "h1", ["alpha alpha"])
_add_doc(rag_conn, "kb_a", "d2", "f", "h2", ["hotel golf"])
ranked = store.search_dense(rag_conn, "kb_a", embed("alpha"), 10)
assert ranked[0][0] == "d1:0" and ranked[0][1] > 0.99
def test_dense_skips_nan_vectors_instead_of_raising(rag_conn):
# A float16 embedder wrote NaN vectors, which vec0 answers with a NULL distance.
_add_doc(rag_conn, "kb_a", "d1", "f", "h1", ["alpha alpha"])
store.create_document(rag_conn, scope = "kb_a", filename = "g", sha256 = "h2", document_id = "d2")
store.add_chunks(rag_conn, "kb_a", "d2", [_chunk("alpha")], [[math.nan] * len(VOCAB)])
ranked = store.search_dense(rag_conn, "kb_a", embed("alpha"), 10)
assert [cid for cid, _ in ranked] == ["d1:0"]
def test_dense_knn_binds_k_rather_than_limit(rag_conn):
"""vec0's KNN bound must arrive as ``k = ?``, not a bare ``LIMIT ?``, which SQLite forwards
to a virtual table's planner only from 3.41 on. CI's SQLite accepts both and returns the
same rows, so nothing else here catches a revert and only the executed statement can say."""
_add_doc(rag_conn, "kb_a", "d1", "f", "h1", ["alpha alpha"])
seen = []
rag_conn.set_trace_callback(seen.append)
try:
store.search_dense(rag_conn, "kb_a", embed("alpha"), 5)
finally:
rag_conn.set_trace_callback(None)
knn = [sql for sql in seen if "chunks_vec" in sql and "MATCH" in sql.upper()]
assert knn, f"search_dense issued no vec0 MATCH query; statements were {seen}"
for sql in knn:
flat = " ".join(sql.split())
assert re.search(r"\bk\s*=", flat), f"vec0 KNN query has no k constraint: {flat}"
assert not re.search(
r"\bLIMIT\b", flat, re.IGNORECASE
), f"vec0 KNN query still leans on LIMIT, which pre-3.41 SQLite never forwards: {flat}"
def test_dense_empty_before_any_ingest(rag_conn):
assert store.search_dense(rag_conn, "kb_a", embed("alpha"), 10) == []
def test_dedupe_by_hash(rag_conn):
_add_doc(rag_conn, "kb_a", "d1", "f", "SHA", ["alpha"])
assert store.document_by_hash(rag_conn, "kb_a", "SHA") == "d1"
assert store.document_by_hash(rag_conn, "kb_a", "OTHER") is None
def test_delete_document_purges_all_tables(rag_conn):
_add_doc(rag_conn, "kb_a", "d1", "f", "h1", ["alpha bravo"])
store.delete_document(rag_conn, "d1")
assert store.search_lexical(rag_conn, "kb_a", "alpha", 10) == []
assert store.search_dense(rag_conn, "kb_a", embed("alpha"), 10) == []
assert store.chunks_by_id(rag_conn, ["d1:0"]) == {}
assert store.get_document(rag_conn, "d1") is None
def test_incremental_add_is_flat(rag_conn):
_add_doc(rag_conn, "kb_a", "d1", "f", "h1", ["alpha bravo charlie"])
before = rag_conn.execute(
"SELECT rowid, chunk_id FROM chunks_fts WHERE scope='kb_a'"
).fetchall()
_add_doc(rag_conn, "kb_a", "d2", "f", "h2", ["delta echo foxtrot"])
after = rag_conn.execute(
"SELECT rowid, chunk_id FROM chunks_fts WHERE scope='kb_a' AND chunk_id LIKE 'd1:%'"
).fetchall()
before_d1 = [(r["rowid"], r["chunk_id"]) for r in before if r["chunk_id"].startswith("d1:")]
after_d1 = [(r["rowid"], r["chunk_id"]) for r in after]
assert before_d1 == after_d1
def test_chunks_by_id_joins_filename(rag_conn):
_add_doc(rag_conn, "kb_a", "d1", "paper.pdf", "h1", ["body text here"])
rows = store.chunks_by_id(rag_conn, ["d1:0"])
assert rows["d1:0"]["filename"] == "paper.pdf"
assert rows["d1:0"]["text"] == "body text here"
def test_kb_crud_and_delete_cascades(rag_conn):
kb_id = store.create_kb(rag_conn, name = "My KB", description = "d", kb_id = "K1")
assert store.get_kb(rag_conn, kb_id)["name"] == "My KB"
assert [k["id"] for k in store.list_kbs(rag_conn)] == ["K1"]
scope = store.kb_scope("K1")
_add_doc(rag_conn, scope, "doc1", "f", "h1", ["alpha bravo"])
store.delete_kb(rag_conn, "K1")
assert store.get_kb(rag_conn, "K1") is None
assert store.list_documents(rag_conn, scope) == []
assert store.search_lexical(rag_conn, scope, "alpha", 10) == []
def test_kb_delete_rolls_back_when_document_cleanup_fails(rag_conn, monkeypatch):
store.create_kb(rag_conn, name = "My KB", kb_id = "K1")
scope = store.kb_scope("K1")
_add_doc(rag_conn, scope, "doc1", "one.txt", "h1", ["alpha bravo"])
_add_doc(rag_conn, scope, "doc2", "two.txt", "h2", ["charlie delta"])
original_delete = store.delete_document
calls = []
def fail_after_delete(
conn,
document_id,
*,
commit = True,
):
calls.append((document_id, commit))
original_delete(conn, document_id, commit = commit)
raise sqlite3.OperationalError("database is busy")
monkeypatch.setattr(store, "delete_document", fail_after_delete)
with pytest.raises(sqlite3.OperationalError, match = "database is busy"):
store.delete_kb(rag_conn, "K1")
assert calls == [("doc1", False)]
assert store.get_kb(rag_conn, "K1") is not None
assert sorted(document["id"] for document in store.list_documents(rag_conn, scope)) == [
"doc1",
"doc2",
]
def _link_folder(
conn,
folder_id,
scope,
path = "/tmp/linked",
):
conn.execute(
"INSERT INTO linked_folders(id, scope_type, scope_id, scope, path, name, "
"auto_sync, status, created_at, updated_at) "
"VALUES(?,?,?,?,?,?,1,'idle','2026-01-01T00:00:00+00:00','2026-01-01T00:00:00+00:00')",
(folder_id, "knowledge_base", scope.removeprefix("kb_"), scope, path, "linked"),
)
conn.commit()
def test_lexical_fast_path_only_while_no_folder_rows_exist(rag_conn):
"""The plain FTS query is used exactly when the filters could not exclude anything."""
_add_doc(rag_conn, "kb_a", "d1", "d1.txt", "h1", ["alpha bravo charlie"])
assert store.linked_folder_rows_exist(rag_conn) is False
_link_folder(rag_conn, "f1", "kb_a")
assert store.linked_folder_rows_exist(rag_conn) is True
rag_conn.execute("DELETE FROM linked_folders")
rag_conn.execute(
"INSERT INTO linked_folder_retired_scopes(scope, retired_at) "
"VALUES('kb_a', '2026-01-01T00:00:00+00:00')"
)
rag_conn.commit()
assert store.linked_folder_rows_exist(rag_conn) is True
def test_lexical_hides_retired_scope_and_unmapped_linked_document(rag_conn):
"""The filters still apply once a folder row exists, fast path or not."""
_add_doc(rag_conn, "kb_a", "d1", "d1.txt", "h1", ["alpha bravo charlie"])
_add_doc(rag_conn, "kb_a", "d2", "d2.txt", "h2", ["alpha delta echo"])
_link_folder(rag_conn, "f1", "kb_a")
rag_conn.execute("UPDATE documents SET linked_folder_id='f1' WHERE id='d2'")
rag_conn.commit()
assert [cid for cid, _ in store.search_lexical(rag_conn, "kb_a", "alpha", 10)] == ["d1:0"]
rag_conn.execute(
"INSERT INTO linked_folder_files(folder_id, relative_path, size_bytes, mtime_ns, "
"document_id, synced_at) VALUES('f1', 'd2.txt', 1, 1, 'd2', "
"'2026-01-01T00:00:00+00:00')"
)
rag_conn.commit()
assert sorted(cid for cid, _ in store.search_lexical(rag_conn, "kb_a", "alpha", 10)) == [
"d1:0",
"d2:0",
]
rag_conn.execute(
"INSERT INTO linked_folder_retired_scopes(scope, retired_at) "
"VALUES('kb_a', '2026-01-01T00:00:00+00:00')"
)
rag_conn.commit()
assert store.search_lexical(rag_conn, "kb_a", "alpha", 10) == []
def test_lexical_results_match_across_both_query_forms(rag_conn):
"""The fast path is a shortcut in work, not in behaviour."""
for i in range(12):
_add_doc(
rag_conn, "kb_a", f"d{i}", f"d{i}.txt", f"h{i}", [f"alpha bravo {'charlie ' * (i % 4)}"]
)
fast = store.search_lexical(rag_conn, "kb_a", "alpha bravo", 5)
_link_folder(rag_conn, "f1", "kb_b")
filtered = store.search_lexical(rag_conn, "kb_a", "alpha bravo", 5)
assert fast == filtered
def test_lexical_gate_and_read_share_one_snapshot(rag_conn, monkeypatch):
"""Gate and FTS read must share one snapshot, so a scope retired in between cannot leak rows."""
from storage import rag_db
_add_doc(rag_conn, "kb_a", "d1", "d1.txt", "h1", ["alpha bravo charlie"])
observed = {}
real = store.linked_folder_rows_exist
def retire_midway(conn):
observed["gate"] = real(conn)
writer = rag_db.get_connection()
try:
writer.execute(
"INSERT INTO linked_folder_retired_scopes(scope, retired_at) "
"VALUES('kb_a', '2026-01-01T00:00:00+00:00')"
)
writer.commit()
finally:
writer.close()
observed["after_commit"] = real(conn)
return observed["gate"]
monkeypatch.setattr(store, "linked_folder_rows_exist", retire_midway)
hits = store.search_lexical(rag_conn, "kb_a", "alpha", 10)
assert observed["gate"] is False
assert observed["after_commit"] is False
assert [cid for cid, _ in hits] == ["d1:0"]
monkeypatch.setattr(store, "linked_folder_rows_exist", real)
assert store.search_lexical(rag_conn, "kb_a", "alpha", 10) == []
def test_lexical_reuses_a_transaction_the_caller_already_opened(rag_conn):
"""A caller holding a transaction keeps its own snapshot, and keeps it open."""
_add_doc(rag_conn, "kb_a", "d1", "d1.txt", "h1", ["alpha bravo charlie"])
rag_conn.execute("BEGIN")
assert [cid for cid, _ in store.search_lexical(rag_conn, "kb_a", "alpha", 10)] == ["d1:0"]
assert rag_conn.in_transaction
rag_conn.commit()
def test_gate_ignores_a_purged_tombstone(rag_conn):
"""The gate must ignore purged tombstones, or every delete would disable the fast path for good."""
_add_doc(rag_conn, "kb_a", "d1", "d1.txt", "h1", ["alpha bravo charlie"])
rag_conn.execute(
"INSERT INTO linked_folder_retired_scopes(scope, retired_at) "
"VALUES('kb_gone', '2026-01-01T00:00:00+00:00')"
)
rag_conn.commit()
assert store.linked_folder_rows_exist(rag_conn) is True
rag_conn.execute(
"UPDATE linked_folder_retired_scopes SET purged_at='2026-01-01T00:00:01+00:00'"
)
rag_conn.commit()
assert store.linked_folder_rows_exist(rag_conn) is False
assert [cid for cid, _ in store.search_lexical(rag_conn, "kb_a", "alpha", 10)] == ["d1:0"]
def test_gate_counts_a_folder_document_that_outlived_its_folder(rag_conn):
"""The gate must count an orphan document that outlived its folder, so unlinked content stays hidden."""
_link_folder(rag_conn, "f1", "kb_a")
store.create_document(
rag_conn,
scope = "kb_a",
filename = "secret.md",
sha256 = "h1",
document_id = "orphan",
linked_folder_id = "f1",
)
store.add_chunks(
rag_conn, "kb_a", "orphan", [_chunk("alpha bravo secret")], [embed("alpha bravo")]
)
assert store.search_lexical(rag_conn, "kb_a", "alpha", 10) == []
rag_conn.execute("DELETE FROM linked_folders WHERE id='f1'")
rag_conn.commit()
assert store.linked_folder_rows_exist(rag_conn) is True
assert store.search_lexical(rag_conn, "kb_a", "alpha", 10) == []
def _pasted_prose(words: int) -> str:
"""Prose words must be purely alphabetic; a letter-digit mix would hit the identifier check."""
letters = "abcdefghijklmnopqrstuvwxyz"
return " ".join(
letters[index % 26]
+ letters[(index // 26) % 26]
+ letters[(index // 676) % 26]
+ letters[(index // 17576) % 26]
+ "qz"
for index in range(words)
)
def test_a_pasted_log_does_not_make_the_archive_query_quadratic(monkeypatch):
"""Query shaping must scan the question once, not per token, or the cost grows quadratically."""
scans = {"n": 0}
real = store._TOKEN
class CountingToken:
def findall(self, text):
scans["n"] += 1
return real.findall(text)
monkeypatch.setattr(store, "_TOKEN", CountingToken())
question = f"what is the current value of ZQXVARA123 {_pasted_prose(2000)}"
expressions = store.conversation_match_queries(question)
assert expressions and expressions[0].startswith('"zqxvara123"')
# Anything growing with token count means the quadratic is back.
assert scans["n"] <= 2, f"tokenized the question {scans['n']} times"
def test_query_shaping_stays_cheap_on_a_pasted_log():
"""Wall-clock companion to the scan count: pasted-log query shaping must stay under 1.0s."""
expressions = assert_linear(
store.conversation_match_queries,
lambda n: f"what is the current value of ZQXVARA123\n{_pasted_prose(n)}",
"paste shaping",
1_500,
)
assert expressions and expressions[0] == '"zqxvara123"'
def test_a_quoted_function_word_survives_the_stopword_filter():
"""Quoted words are not stopwords: a quoted word must survive the filter that drops it unquoted."""
quoted = store.conversation_match_queries('What did I say about "this"?')
plain = store.conversation_match_queries("What did I say about this?")
assert quoted == ['"say" OR "this"']
assert plain == ['"say"']
assert len(quoted) == 1
def test_the_candidate_window_is_cut_in_conversation_order(rag_home, rag_conn):
"""Candidate windows are cut in conversation order; cutting by chunk id could drop both true ends."""
import types
from core.rag import store
conn = rag_conn
scope = "convarchive_legacy"
documents, per_document = 100, 3
position_of = {}
for position in range(documents):
document_id = f"{(position + documents // 2) % documents:04d}-turn"
position_of[document_id] = position
store.create_document(
conn,
scope = scope,
thread_id = "t",
filename = "earlier turn",
sha256 = f"h{position}",
status = "completed",
embedding_model = "m",
archive_messages = 2,
archive_ordinal = None,
document_id = document_id,
created_at = "2026-01-01T00:00:00+00:00",
commit = False,
)
chunks = [
types.SimpleNamespace(
chunk_index = index,
text = "ZQXTIEBREAK legacy turn statement",
page_number = None,
source_page_index = None,
token_count = 5,
char_count = 20,
)
for index in range(per_document)
]
store.add_chunks(conn, scope, document_id, chunks, [[0.0] * 4] * per_document)
conn.commit()
everything = store.search_lexical(conn, scope, "ZQXTIEBREAK", documents * per_document + 10)
assert len(everything) == documents * per_document
assert len({score for _, score in everything}) == 1
half = 128
oldest = [
c for c, _ in store.search_lexical(conn, scope, "ZQXTIEBREAK", half, oldest_first = True)
]
newest = [
c for c, _ in store.search_lexical(conn, scope, "ZQXTIEBREAK", half, newest_first = True)
]
in_oldest = sorted({position_of[chunk.rsplit(":", 1)[0]] for chunk in oldest})
in_newest = sorted({position_of[chunk.rsplit(":", 1)[0]] for chunk in newest})
assert 0 in in_oldest, in_oldest
assert documents - 1 in in_newest, in_newest
assert in_oldest[0] == 0 and in_oldest == list(range(len(in_oldest))), in_oldest
assert in_newest[-1] == documents - 1, in_newest
assert in_newest == list(range(documents - len(in_newest), documents)), in_newest
assert not set(in_oldest) & set(in_newest)
def test_the_candidate_order_survives_a_re_embed(rag_home, rag_conn):
"""A re-embed must keep the rowid that the candidate ORDER BY sorts on, else order silently changes."""
import types
from core.rag import store
conn = rag_conn
scope = "convarchive_reembed"
turns = 5
def _document_id(position):
return f"{turns - position:04d}-turn"
def _write(
position,
model,
*,
rowid = None,
created = None,
ordinal = None,
):
store.create_document(
conn,
scope = scope,
thread_id = "t",
filename = "earlier turn",
sha256 = f"h{position}",
status = "completed",
embedding_model = model,
archive_messages = 2,
archive_ordinal = ordinal,
document_id = _document_id(position),
# One tick for every turn, so only rowid records order.
created_at = created or "2026-01-01T00:00:00+00:00",
rowid = rowid,
commit = False,
)
store.add_chunks(
conn,
scope,
_document_id(position),
[
types.SimpleNamespace(
chunk_index = 0,
text = "ZQXREEMBED legacy turn statement",
page_number = None,
source_page_index = None,
token_count = 5,
char_count = 20,
)
],
[[0.0] * 4],
)
for position in range(turns):
_write(position, "old-model")
conn.commit()
def _positions(**direction):
hits = store.search_lexical(conn, scope, "ZQXREEMBED", turns + 10, **direction)
return [turns - int(chunk.rsplit(":", 1)[0].split("-")[0]) for chunk, _s in hits]
assert _positions(oldest_first = True) == list(range(turns))
rowids_before = dict(
conn.execute("SELECT id, rowid FROM documents WHERE scope=?", (scope,)).fetchall()
)
for position in [4, 3, 2]:
identity = store.document_rewrite_identity(conn, _document_id(position)) or {}
store.delete_document(conn, _document_id(position), commit = False)
_write(
position,
"new-model",
rowid = identity.get("rowid"),
created = identity.get("created_at"),
ordinal = identity.get("archive_ordinal"),
)
conn.commit()
# The premise, and the test is vacuous without it: the rows really were replaced.
assert {
row[0]
for row in conn.execute("SELECT embedding_model FROM documents WHERE scope=?", (scope,))
} == {"old-model", "new-model"}
assert (
dict(conn.execute("SELECT id, rowid FROM documents WHERE scope=?", (scope,)).fetchall())
== rowids_before
)
assert _positions(oldest_first = True) == list(range(turns))
assert _positions(newest_first = True) == list(reversed(range(turns)))
def test_a_legacy_archive_still_gets_two_different_ends(rag_home, rag_conn):
"""Legacy NULL-ordinal archives must still give two distinct window ends, not one repeated query."""
import types
from core.rag import store
conn = rag_conn
scope = "convarchive_legacy"
for index in range(8):
document = store.create_document(
conn,
scope = scope,
thread_id = "t",
filename = f"earlier turn {index}",
sha256 = f"h{index}",
status = "completed",
embedding_model = "m",
archive_messages = 2,
archive_ordinal = None,
commit = False,
)
chunk = types.SimpleNamespace(
chunk_index = 0,
text = f"ZQXLEGACY token number {index}",
page_number = None,
source_page_index = None,
token_count = 5,
char_count = 20,
)
store.add_chunks(conn, scope, document, [chunk], [[0.0, 0.0, 0.0, 0.0]])
conn.commit()
oldest = [
chunk for chunk, _ in store.search_lexical(conn, scope, "ZQXLEGACY", 3, oldest_first = True)
]
newest = [
chunk for chunk, _ in store.search_lexical(conn, scope, "ZQXLEGACY", 3, newest_first = True)
]
assert oldest and newest
assert oldest != newest, "both ends of the fetch returned the same rows"
assert not set(oldest) & set(newest), (oldest, newest)
def test_dense_packs_float32_rows_without_struct_and_the_query_once(rag_conn, monkeypatch):
np = pytest.importorskip("numpy")
# vec0 answers both searches: the first one queues the in-memory cache build (#15961) on a background
# thread, and a cache that lands between them scores in numpy, a few ulps away from vec0.
monkeypatch.setenv("RAG_VEC_CACHE", "0")
texts = ["alpha alpha", "hotel golf", "alpha bravo"]
for i, scope in enumerate(("kb_a", "kb_b", "kb_c")):
doc = f"d{i}"
store.create_document(rag_conn, scope = scope, filename = "f", sha256 = doc, document_id = doc)
rows = np.asarray([embed(t) for t in texts], dtype = np.float32)
store.add_chunks(rag_conn, scope, doc, [_chunk(t, j) for j, t in enumerate(texts)], rows)
stored = rag_conn.execute("SELECT embedding FROM chunks_vec ORDER BY rowid").fetchall()
expected = [
store.struct.pack(f"{len(VOCAB)}f", *np.asarray(embed(t), dtype = np.float32).tolist())
for t in texts
] * 3
assert [bytes(r[0]) for r in stored] == expected
packs = []
real_pack = store.struct.pack
monkeypatch.setattr(store.struct, "pack", lambda *a: packs.append(a) or real_pack(*a))
query = np.asarray(embed("alpha"), dtype = np.float32)
ranked = store.search_dense(rag_conn, ["kb_a", "kb_b", "kb_c"], query, 9)
as_list = store.search_dense(rag_conn, ["kb_a", "kb_b", "kb_c"], query.tolist(), 9)
assert ranked == as_list and ranked[0][1] > 0.99
assert len(packs) == 1