Skip to content

Personal AI, On Personal Devices

Python

feat: add content-addressed AttachmentStore with SHA-256 dedup

krypticmouse99d2013d519c
diff --git a/src/openjarvis/connectors/attachment_store.py b/src/openjarvis/connectors/attachment_store.py
new file mode 100644
index 00000000..7b653562
--- /dev/null
+++ b/src/openjarvis/connectors/attachment_store.py
@@ -0,0 +1,175 @@
+"""AttachmentStore — content-addressed blob storage for Deep Research attachments.
+
+Stores binary attachments at ``{base_dir}/{sha256[:2]}/{sha256}`` and tracks
+metadata in a SQLite database at ``{base_dir}/attachments.db``.
+
+Deduplication is automatic: re-storing the same bytes accumulates source_doc_ids
+but does not write a second copy of the file.
+
+Pure Python ``sqlite3`` (no Rust extension required).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import sqlite3
+import time
+from pathlib import Path
+from typing import Dict, List, Optional
+
+# ---------------------------------------------------------------------------
+# DDL
+# ---------------------------------------------------------------------------
+
+_CREATE_TABLE = """
+CREATE TABLE IF NOT EXISTS attachments (
+ sha256 TEXT PRIMARY KEY,
+ filename TEXT NOT NULL DEFAULT '',
+ mime_type TEXT NOT NULL DEFAULT '',
+ size_bytes INTEGER NOT NULL DEFAULT 0,
+ source_doc_ids TEXT NOT NULL DEFAULT '[]',
+ created_at REAL NOT NULL
+);
+"""
+
+# ---------------------------------------------------------------------------
+# AttachmentStore
+# ---------------------------------------------------------------------------
+
+
+class AttachmentStore:
+ """Content-addressed blob store with SQLite metadata index.
+
+ Files are written to ``{base_dir}/{sha256[:2]}/{sha256}`` so that the
+ directory fan-out stays bounded even for millions of blobs. The SQLite
+ metadata table tracks filename, MIME type, size, and the list of
+ source document IDs that reference each blob.
+ """
+
+ def __init__(self, base_dir: str = "") -> None:
+ if not base_dir:
+ from openjarvis.core.config import DEFAULT_CONFIG_DIR
+
+ base_dir = str(DEFAULT_CONFIG_DIR / "blobs")
+
+ self._base_dir = Path(base_dir)
+ self._base_dir.mkdir(parents=True, exist_ok=True)
+
+ db_path = self._base_dir / "attachments.db"
+ self._conn = sqlite3.connect(str(db_path), check_same_thread=False)
+ self._conn.row_factory = sqlite3.Row
+ self._setup()
+
+ # ------------------------------------------------------------------
+ # Internal setup
+ # ------------------------------------------------------------------
+
+ def _setup(self) -> None:
+ self._conn.execute("PRAGMA journal_mode=WAL;")
+ self._conn.execute(f"{_CREATE_TABLE}")
+ self._conn.commit()
+
+ # ------------------------------------------------------------------
+ # Public API
+ # ------------------------------------------------------------------
+
+ def store(
+ self,
+ content: bytes,
+ *,
+ filename: str,
+ mime_type: str = "",
+ source_doc_id: str = "",
+ ) -> str:
+ """Store *content* and return its SHA-256 hex digest.
+
+ The call is idempotent with respect to the blob file — if the same
+ bytes are stored again only the metadata (source_doc_ids) is updated.
+ """
+ sha = hashlib.sha256(content).hexdigest()
+
+ # Write blob file (idempotent)
+ blob_dir = self._base_dir / sha[:2]
+ blob_dir.mkdir(parents=True, exist_ok=True)
+ blob_path = blob_dir / sha
+ if not blob_path.exists():
+ blob_path.write_bytes(content)
+
+ # Upsert metadata row
+ existing = self._conn.execute(
+ "SELECT source_doc_ids FROM attachments WHERE sha256 = ?", (sha,)
+ ).fetchone()
+
+ if existing is None:
+ source_ids: List[str] = []
+ if source_doc_id:
+ source_ids.append(source_doc_id)
+ self._conn.execute(
+ """
+ INSERT INTO attachments
+ (sha256, filename, mime_type, size_bytes,
+ source_doc_ids, created_at)
+ VALUES (?, ?, ?, ?, ?, ?)
+ """,
+ (
+ sha,
+ filename,
+ mime_type,
+ len(content),
+ json.dumps(source_ids),
+ time.time(),
+ ),
+ )
+ else:
+ source_ids = json.loads(existing["source_doc_ids"])
+ if source_doc_id and source_doc_id not in source_ids:
+ source_ids.append(source_doc_id)
+ self._conn.execute(
+ "UPDATE attachments SET source_doc_ids = ? WHERE sha256 = ?",
+ (json.dumps(source_ids), sha),
+ )
+
+ self._conn.commit()
+ return sha
+
+ def get_metadata(self, sha: str) -> Optional[Dict]:
+ """Return the metadata dict for *sha*, or ``None`` if not found.
+
+ Returned keys: ``sha256``, ``filename``, ``mime_type``,
+ ``size_bytes``, ``source_doc_ids`` (list), ``created_at``.
+ """
+ row = self._conn.execute(
+ "SELECT * FROM attachments WHERE sha256 = ?", (sha,)
+ ).fetchone()
+ if row is None:
+ return None
+ return {
+ "sha256": row["sha256"],
+ "filename": row["filename"],
+ "mime_type": row["mime_type"],
+ "size_bytes": row["size_bytes"],
+ "source_doc_ids": json.loads(row["source_doc_ids"]),
+ "created_at": row["created_at"],
+ }
+
+ def get_content(self, sha: str) -> Optional[bytes]:
+ """Return the raw bytes for *sha*, or ``None`` if the blob is missing."""
+ blob_path = self._base_dir / sha[:2] / sha
+ if not blob_path.exists():
+ return None
+ return blob_path.read_bytes()
+
+ # ------------------------------------------------------------------
+ # Helpers
+ # ------------------------------------------------------------------
+
+ def close(self) -> None:
+ """Close the underlying SQLite connection."""
+ try:
+ self._conn.close()
+ except Exception:
+ pass
+
+
+__all__ = ["AttachmentStore"]
diff --git a/tests/connectors/test_attachment_store.py b/tests/connectors/test_attachment_store.py
new file mode 100644
index 00000000..562e8ff9
--- /dev/null
+++ b/tests/connectors/test_attachment_store.py
@@ -0,0 +1,102 @@
+"""Tests for AttachmentStore — content-addressed blob storage."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from openjarvis.connectors.attachment_store import AttachmentStore
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+def store(tmp_path: Path) -> AttachmentStore:
+ return AttachmentStore(base_dir=str(tmp_path / "blobs"))
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+def test_store_returns_sha256(store: AttachmentStore) -> None:
+ """store() returns a 64-character lowercase hex string (SHA-256)."""
+ sha = store.store(b"hello world", filename="hello.txt")
+ assert isinstance(sha, str)
+ assert len(sha) == 64
+ assert sha == sha.lower()
+ # Only hex characters
+ int(sha, 16)
+
+
+def test_store_creates_blob_file(store: AttachmentStore, tmp_path: Path) -> None:
+ """Blob file is written at {base_dir}/{sha[:2]}/{sha} with correct content."""
+ data = b"binary content for blob test"
+ sha = store.store(data, filename="blob.bin", mime_type="application/octet-stream")
+
+ blob_path = tmp_path / "blobs" / sha[:2] / sha
+ assert blob_path.exists(), f"Expected blob at {blob_path}"
+ assert blob_path.read_bytes() == data
+
+
+def test_dedup_same_content(store: AttachmentStore) -> None:
+ """Storing identical bytes twice returns the same hash."""
+ data = b"duplicate content"
+ sha1 = store.store(data, filename="file1.txt")
+ sha2 = store.store(data, filename="file2.txt")
+ assert sha1 == sha2
+
+
+def test_dedup_tracks_multiple_sources(store: AttachmentStore) -> None:
+ """Three stores of the same bytes accumulate three distinct source_doc_ids."""
+ data = b"shared attachment content"
+ store.store(data, filename="attach.pdf", source_doc_id="doc:001")
+ store.store(data, filename="attach.pdf", source_doc_id="doc:002")
+ store.store(data, filename="attach.pdf", source_doc_id="doc:003")
+
+ sha = store.store(data, filename="attach.pdf") # 4th call, no new source
+ meta = store.get_metadata(sha)
+ assert meta is not None
+ source_ids = meta["source_doc_ids"]
+ assert "doc:001" in source_ids
+ assert "doc:002" in source_ids
+ assert "doc:003" in source_ids
+ assert len(source_ids) == 3 # no duplicates
+
+
+def test_get_metadata(store: AttachmentStore) -> None:
+ """get_metadata() returns correct filename, mime_type, and size_bytes."""
+ data = b"metadata test payload"
+ sha = store.store(
+ data,
+ filename="report.pdf",
+ mime_type="application/pdf",
+ source_doc_id="doc:meta:001",
+ )
+
+ meta = store.get_metadata(sha)
+ assert meta is not None
+ assert meta["sha256"] == sha
+ assert meta["filename"] == "report.pdf"
+ assert meta["mime_type"] == "application/pdf"
+ assert meta["size_bytes"] == len(data)
+ assert "created_at" in meta
+ assert isinstance(meta["source_doc_ids"], list)
+
+
+def test_get_content(store: AttachmentStore) -> None:
+ """get_content() returns the exact bytes that were stored."""
+ data = b"\x00\x01\x02\x03binary\xff\xfe"
+ sha = store.store(data, filename="binary.bin")
+ retrieved = store.get_content(sha)
+ assert retrieved == data
+
+
+def test_get_content_nonexistent(store: AttachmentStore) -> None:
+ """get_content() returns None for an unknown SHA-256."""
+ fake_sha = "a" * 64
+ assert store.get_content(fake_sha) is None