DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
"""Tests for the Persistent File Library feature (issue #1437)."""
from __future__ import annotations
import asyncio
import hashlib
from pathlib import Path
import tempfile
import pytest
from deeptutor.services.storage.file_library import (
FileLibraryStore,
get_file_library_store,
reset_file_library_store,
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
def tmp_db_path(tmp_path: Path) -> Path:
return tmp_path / "file_library.db"
@pytest.fixture
def store(tmp_db_path: Path) -> FileLibraryStore:
reset_file_library_store()
return FileLibraryStore(db_path=tmp_db_path)
@pytest.fixture
def rooted_store(tmp_db_path: Path, tmp_path: Path) -> FileLibraryStore:
"""A store with an explicit library root under tmp_path.
``_delete_file`` prunes empty parent directories up to (but not
including) the root, so the tests need a root they can inspect.
"""
reset_file_library_store()
root = tmp_path / "library-files"
return FileLibraryStore(db_path=tmp_db_path, root=root)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _sha256(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def _content(data: bytes) -> tuple[str, str]:
"""Return (sha256, base_name) for the data."""
return _sha256(data), "file.pdf"
# ---------------------------------------------------------------------------
# add_file
# ---------------------------------------------------------------------------
def test_add_file_returns_entry(store: FileLibraryStore) -> None:
"""add_file must return a complete library entry."""
data = b"Hello, World!"
entry = asyncio.run(store.add_file(data=data, filename="hello.txt", mime_type="text/plain"))
assert entry["id"] is not None
assert entry["sha256"] == _sha256(data)
assert entry["filename"] == "hello.txt"
assert entry["mime_type"] == "text/plain"
assert entry["size_bytes"] == len(data)
assert entry["is_deleted"] is False
def test_add_file_persists_to_disk(store: FileLibraryStore) -> None:
"""The file bytes must be stored on disk under the library root."""
data = b"Test content"
entry = asyncio.run(store.add_file(data=data, filename="test.txt", mime_type="text/plain"))
resolved = store.resolve_path(entry["id"])
assert resolved is not None
assert resolved.read_bytes() == data
def test_add_file_deduplicates_by_hash(store: FileLibraryStore) -> None:
"""Uploading identical content must return the existing entry, not create a duplicate."""
data = b"Duplicate content"
e1 = asyncio.run(
store.add_file(data=data, filename="original.pdf", mime_type="application/pdf")
)
e2 = asyncio.run(store.add_file(data=data, filename="copy.pdf", mime_type="application/pdf"))
assert e1["id"] == e2["id"]
assert e1["sha256"] == e2["sha256"]
def test_add_file_different_content_gives_different_hash(store: FileLibraryStore) -> None:
"""Different byte content must produce different hashes."""
e1 = asyncio.run(store.add_file(data=b"Content A", filename="a.txt", mime_type="text/plain"))
e2 = asyncio.run(store.add_file(data=b"Content B", filename="b.txt", mime_type="text/plain"))
assert e1["sha256"] != e2["sha256"]
assert e1["id"] != e2["id"]
def test_add_file_concurrent_uploads_of_same_content_do_not_duplicate(
store: FileLibraryStore,
) -> None:
"""Regression test: concurrent uploads of identical content used to all
miss the dedup SELECT before any of them committed its INSERT (a TOCTOU
race across the OS threads ``asyncio.to_thread`` dispatches to), each
creating its own duplicate row and file on disk instead of deduplicating.
"""
import threading
data = b"same content uploaded from multiple tabs at once"
n_threads = 8
barrier = threading.Barrier(n_threads)
results: list[dict] = [None] * n_threads # type: ignore[list-item]
def upload(i: int) -> None:
barrier.wait()
results[i] = store._add_file_sync(data, f"upload-{i}.txt", "text/plain")
threads = [threading.Thread(target=upload, args=(i,)) for i in range(n_threads)]
for t in threads:
t.start()
for t in threads:
t.join()
ids = {r["id"] for r in results}
assert len(ids) == 1, f"expected a single deduplicated entry, got {len(ids)}: {ids}"
entries = asyncio.run(store.list_files())
assert len(entries) == 1
# ---------------------------------------------------------------------------
# list_files
# ---------------------------------------------------------------------------
def test_list_files_returns_all_active(store: FileLibraryStore) -> None:
"""list_files must return every non-deleted entry, newest first."""
# Track IDs and creation times of files we create to verify relative ordering.
own_ids = []
own_created_ats = []
for i in range(5):
e = asyncio.run(
store.add_file(
data=f"Content {i}".encode(), filename=f"file{i}.txt", mime_type="text/plain"
)
)
own_ids.append(e["id"])
own_created_ats.append(e["created_at"])
entries = asyncio.run(store.list_files())
# Among the entries we created, verify newest-first ordering using timestamps.
own_results = [e for e in entries if e["id"] in own_ids]
assert len(own_results) == 5
result_created_ats = [e["created_at"] for e in own_results]
# Sort by created_at DESC to get the expected order.
expected_order = [eid for _, eid in sorted(zip(own_created_ats, own_ids), reverse=True)]
assert [e["id"] for e in own_results] == expected_order
def test_list_files_respects_limit_offset(store: FileLibraryStore) -> None:
"""limit and offset must paginate the result list."""
for i in range(5):
asyncio.run(
store.add_file(data=f"Data {i}".encode(), filename=f"f{i}.txt", mime_type="text/plain")
)
page1 = asyncio.run(store.list_files(limit=2, offset=0))
page2 = asyncio.run(store.list_files(limit=2, offset=2))
assert len(page1) == 2
assert len(page2) == 2
assert page1[0]["id"] != page2[0]["id"]
def test_list_files_excludes_deleted(store: FileLibraryStore) -> None:
"""Soft-deleted entries must not appear in list_files."""
e1 = asyncio.run(store.add_file(data=b"Keep me", filename="keep.txt", mime_type="text/plain"))
e2 = asyncio.run(
store.add_file(data=b"Delete me", filename="delete.txt", mime_type="text/plain")
)
asyncio.run(store.delete_file(e2["id"]))
entries = asyncio.run(store.list_files())
ids = [e["id"] for e in entries]
assert e1["id"] in ids
assert e2["id"] not in ids
# ---------------------------------------------------------------------------
# search_files
# ---------------------------------------------------------------------------
def test_search_files_by_filename(store: FileLibraryStore) -> None:
"""search_files must match on filename."""
asyncio.run(
store.add_file(
data=b"Content", filename="project_proposal.pdf", mime_type="application/pdf"
)
)
asyncio.run(store.add_file(data=b"Content", filename="notes.txt", mime_type="text/plain"))
results = asyncio.run(store.search_files("project"))
assert len(results) == 1
assert results[0]["filename"] == "project_proposal.pdf"
def test_search_files_no_match_returns_empty(store: FileLibraryStore) -> None:
"""A query that matches nothing must return an empty list."""
asyncio.run(store.add_file(data=b"Content", filename="other.txt", mime_type="text/plain"))
results = asyncio.run(store.search_files("nonexistent_term_xyz"))
assert results == []
def test_search_files_excludes_deleted(store: FileLibraryStore) -> None:
"""Deleted entries must not appear in search results."""
e = asyncio.run(
store.add_file(data=b"Important document", filename="doc.pdf", mime_type="application/pdf")
)
asyncio.run(store.delete_file(e["id"]))
results = asyncio.run(store.search_files("Important"))
assert results == []
# ---------------------------------------------------------------------------
# delete_file (soft delete)
# ---------------------------------------------------------------------------
def test_delete_file_soft_deletes(store: FileLibraryStore) -> None:
"""delete_file must set is_deleted=True without removing the row."""
e = asyncio.run(
store.add_file(data=b"Content", filename="todelete.txt", mime_type="text/plain")
)
result = asyncio.run(store.delete_file(e["id"]))
assert result is True
entry = asyncio.run(store.get_file(e["id"]))
assert entry is not None
assert entry["is_deleted"] is True
def test_delete_file_idempotent(store: FileLibraryStore) -> None:
"""Deleting an already-deleted entry must return True without error."""
e = asyncio.run(store.add_file(data=b"Content", filename="file.txt", mime_type="text/plain"))
asyncio.run(store.delete_file(e["id"]))
result = asyncio.run(store.delete_file(e["id"]))
assert result is True
def test_delete_file_nonexistent_returns_false(store: FileLibraryStore) -> None:
"""Deleting a non-existent ID must return False."""
result = asyncio.run(store.delete_file("nonexistent_id"))
assert result is False
# ---------------------------------------------------------------------------
# hard_delete_file
# ---------------------------------------------------------------------------
def test_hard_delete_file_removes_row(store: FileLibraryStore) -> None:
"""hard_delete_file must physically remove the row from the database."""
e = asyncio.run(
store.add_file(data=b"Content", filename="permanent.txt", mime_type="text/plain")
)
asyncio.run(store.delete_file(e["id"])) # must be soft-deleted first
result = asyncio.run(store.hard_delete_file(e["id"]))
assert result is True
entry = asyncio.run(store.get_file(e["id"]))
assert entry is None
def test_hard_delete_file_without_soft_delete_fails(store: FileLibraryStore) -> None:
"""hard_delete_file must refuse to delete a non-deleted entry."""
e = asyncio.run(store.add_file(data=b"Content", filename="file.txt", mime_type="text/plain"))
result = asyncio.run(store.hard_delete_file(e["id"]))
assert result is False
# ---------------------------------------------------------------------------
# _delete_file — disk cleanup and failure paths (DT-22)
#
# Locks in the current behaviour of ``FileLibraryStore._delete_file``:
# the on-disk cleanup is best-effort. A failed ``unlink`` is only logged,
# and any exception raised while pruning now-empty parent directories is
# swallowed entirely by a broad ``except Exception: pass`` around the
# parent-traversal loop. These tests pin that behaviour so a later fix
# that starts surfacing (or retrying) disk errors must consciously update
# them — and so the known list-vs-disk inconsistency window stays visible
# instead of rotting silently.
# ---------------------------------------------------------------------------
def test_delete_file_removes_target_and_empty_parent_dirs(
rooted_store: FileLibraryStore,
) -> None:
"""A successful delete removes the file and prunes now-empty parent
directories, stopping at (and keeping) the library root."""
store = rooted_store
root = store._root
library_path = "sub/deep/report.txt"
store._write_file(library_path, b"payload")
assert (root / "sub" / "deep" / "report.txt").is_file()
store._delete_file(library_path)
assert not (root / "sub" / "deep" / "report.txt").exists()
assert not (root / "sub" / "deep").exists() # emptied parents pruned
assert not (root / "sub").exists()
assert root.is_dir() # root itself is never removed
def test_delete_file_keeps_nonempty_parent_dirs(rooted_store: FileLibraryStore) -> None:
"""Parent pruning must stop at directories that still hold other files."""
store = rooted_store
root = store._root
store._write_file("batch/keep.txt", b"keep")
store._write_file("batch/gone.txt", b"gone")
store._delete_file("batch/gone.txt")
assert not (root / "batch" / "gone.txt").exists()
assert (root / "batch" / "keep.txt").is_file() # untouched sibling
assert (root / "batch").is_dir() # non-empty parent kept
def test_delete_file_missing_target_is_silent_noop(rooted_store: FileLibraryStore) -> None:
"""A library_path that does not exist on disk must be a silent no-op:
no exception, no directories created, nothing written."""
store = rooted_store
root = store._root
store._delete_file("never-existed.txt")
assert not (root / "never-existed.txt").exists()
if root.is_dir():
assert list(root.iterdir()) == [] # nothing materialised on disk
def test_hard_delete_succeeds_when_disk_file_already_missing(
rooted_store: FileLibraryStore,
) -> None:
"""DB-vs-disk sync: when the disk file has already vanished (e.g. pruned
out-of-band), the hard-delete flow must still complete — the row is
removed and the missing disk file is tolerated as a silent no-op."""
store = rooted_store
entry = store._add_file_sync(b"orphan payload", "orphan.txt", "text/plain")
disk_path = store._file_path(entry["library_path"])
assert disk_path.is_file()
disk_path.unlink() # file disappears out-of-band
assert asyncio.run(store.delete_file(entry["id"])) is True
assert asyncio.run(store.hard_delete_file(entry["id"])) is True
assert asyncio.run(store.get_file(entry["id"])) is None # row gone
def test_delete_file_unlink_failure_only_logs_warning(
rooted_store: FileLibraryStore,
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
"""When ``unlink`` fails with OSError the failure is only logged — the
file stays on disk and no exception propagates. This pins the known
best-effort gap: the DB row can be dropped while the file survives
(list-vs-disk inconsistency window flagged by the DT-22 scan)."""
store = rooted_store
root = store._root
library_path = "stuck/report.txt"
store._write_file(library_path, b"payload")
target = root / "stuck" / "report.txt"
original_unlink = Path.unlink
def failing_unlink(self: Path, missing_ok: bool = False) -> None:
if self == target:
raise PermissionError(13, f"simulated denial for {self}")
original_unlink(self, missing_ok=missing_ok)
monkeypatch.setattr(Path, "unlink", failing_unlink)
with caplog.at_level("WARNING", logger="deeptutor.services.storage.file_library"):
store._delete_file(library_path) # must not raise
monkeypatch.undo()
assert target.is_file() # unlink failed -> file still on disk
assert "failed to delete library file" in caplog.text
assert str(target) in caplog.text
def test_delete_file_parent_traversal_exception_is_swallowed(
rooted_store: FileLibraryStore,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""DT-22: an exception raised while pruning empty parent directories is
swallowed by the broad ``except Exception: pass`` around the traversal
loop. Current behaviour: the target file is still unlinked, the
remaining parent cleanup is aborted mid-loop (empty directories are
left behind), and no error surfaces to the caller."""
store = rooted_store
root = store._root
library_path = "locked/report.txt"
store._write_file(library_path, b"payload")
locked_dir = root / "locked"
original_iterdir = Path.iterdir
def raising_iterdir(self: Path):
if self == locked_dir:
raise PermissionError(13, f"simulated denial for {self}")
return original_iterdir(self)
monkeypatch.setattr(Path, "iterdir", raising_iterdir)
store._delete_file(library_path) # must not raise
monkeypatch.undo()
assert not (root / "locked" / "report.txt").exists() # file still removed
assert locked_dir.is_dir() # traversal aborted -> empty dir left behind
# ---------------------------------------------------------------------------
# restore_file
# ---------------------------------------------------------------------------
def test_restore_file_clears_deleted_flag(store: FileLibraryStore) -> None:
"""restore_file must set is_deleted=False."""
e = asyncio.run(
store.add_file(data=b"Content", filename="restorable.txt", mime_type="text/plain")
)
asyncio.run(store.delete_file(e["id"]))
result = asyncio.run(store.restore_file(e["id"]))
assert result is True
entry = asyncio.run(store.get_file(e["id"]))
assert entry is not None
assert entry["is_deleted"] is False
def test_restore_file_idempotent(store: FileLibraryStore) -> None:
"""Restoring an active entry must return True without error."""
e = asyncio.run(store.add_file(data=b"Content", filename="active.txt", mime_type="text/plain"))
result = asyncio.run(store.restore_file(e["id"]))
assert result is True
# ---------------------------------------------------------------------------
# get_file
# ---------------------------------------------------------------------------
def test_get_file_returns_entry(store: FileLibraryStore) -> None:
"""get_file must return the full entry dict for an existing ID."""
e = asyncio.run(store.add_file(data=b"Content", filename="getme.txt", mime_type="text/plain"))
fetched = asyncio.run(store.get_file(e["id"]))
assert fetched is not None
assert fetched["id"] == e["id"]
assert fetched["filename"] == "getme.txt"
def test_get_file_deleted_returns_entry_with_flag(store: FileLibraryStore) -> None:
"""get_file must still return the entry (with is_deleted=True) for soft-deleted files."""
e = asyncio.run(store.add_file(data=b"Content", filename="deleted.txt", mime_type="text/plain"))
asyncio.run(store.delete_file(e["id"]))
fetched = asyncio.run(store.get_file(e["id"]))
assert fetched is not None
assert fetched["is_deleted"] is True
def test_get_file_nonexistent_returns_none(store: FileLibraryStore) -> None:
"""get_file must return None for a non-existent ID."""
result = asyncio.run(store.get_file("does_not_exist"))
assert result is None
# ---------------------------------------------------------------------------
# resolve_path
# ---------------------------------------------------------------------------
def test_resolve_path_returns_existing_file(store: FileLibraryStore) -> None:
"""resolve_path must return the Path to the stored file for a valid ID."""
e = asyncio.run(
store.add_file(data=b"PDF content here", filename="doc.pdf", mime_type="application/pdf")
)
path = store.resolve_path(e["id"])
assert path is not None
assert path.exists()
assert path.read_bytes() == b"PDF content here"
def test_resolve_path_deleted_returns_none(store: FileLibraryStore) -> None:
"""resolve_path must return None for a soft-deleted entry."""
e = asyncio.run(
store.add_file(data=b"Content", filename="deleted_file.txt", mime_type="text/plain")
)
asyncio.run(store.delete_file(e["id"]))
path = store.resolve_path(e["id"])
assert path is None
def test_resolve_path_nonexistent_returns_none(store: FileLibraryStore) -> None:
"""resolve_path must return None for a non-existent ID."""
path = store.resolve_path("no_such_id")
assert path is None
# ---------------------------------------------------------------------------
# Singleton / reset
# ---------------------------------------------------------------------------
def test_get_file_library_store_returns_singleton(store: FileLibraryStore) -> None:
"""get_file_library_store must return the process-wide singleton."""
reset_file_library_store()
s1 = get_file_library_store()
s2 = get_file_library_store()
assert s1 is s2
def test_reset_clears_singleton() -> None:
"""reset_file_library_store must clear the singleton so a new instance is created."""
reset_file_library_store()
s1 = get_file_library_store()
reset_file_library_store()
s2 = get_file_library_store()
assert s1 is not s2