YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
// screenpipe — AI that knows everything you've seen, said, or heard
// https://screenpipe.com
use super::{Value, TABLES};
use crate::storage::{storage_error, HybridStorage};
use libsqlite3_sys as ffi;
use sqlx::{sqlite::SqlitePoolOptions, SqliteConnection};
use std::{
collections::HashSet,
ffi::{c_int, c_void, CStr},
sync::{atomic::Ordering, Arc, Mutex},
};
#[derive(Default)]
struct Statements {
active: HashSet<usize>,
lease: Option<StatementLease>,
}
struct StatementLease(Arc<HybridStorage>);
impl StatementLease {
fn new(storage: &Arc<HybridStorage>) -> Self {
storage.bulk.statements.fetch_add(1, Ordering::SeqCst);
Self(Arc::clone(storage))
}
}
impl Drop for StatementLease {
fn drop(&mut self) {
self.0.bulk.statements.fetch_sub(1, Ordering::SeqCst);
}
}
impl HybridStorage {
/// Called after retiring locators under the writer permit. A snapshot
/// admitted after this cutoff sees current files; existing snapshots pin
/// their old files through statement completion or transaction end.
pub(crate) fn sql_readers_active(&self) -> bool {
self.bulk.statements.load(Ordering::SeqCst) != 0
}
}
struct Context {
storage: Arc<HybridStorage>,
database: usize,
statements: Mutex<Statements>,
}
pub(crate) fn pool_options(
storage: Option<Arc<HybridStorage>>,
logical: bool,
) -> SqlitePoolOptions {
let options = crate::write_queue::capture_pool_options();
let Some(storage) = storage.filter(|s| s.has_bulk()) else {
return options;
};
options.after_connect(move |conn, _| {
let storage = Arc::clone(&storage);
Box::pin(async move { register(conn, storage, logical).await })
})
}
pub(crate) async fn register(
conn: &mut SqliteConnection,
storage: Arc<HybridStorage>,
logical: bool,
) -> Result<(), sqlx::Error> {
register_hash(conn).await?;
super::elements::register(conn, Arc::clone(&storage)).await?;
{
let mut locked = conn.lock_handle().await?;
let db = locked.as_raw_handle().as_ptr();
let context = Box::into_raw(Box::new(Context {
storage,
database: db as usize,
statements: Mutex::new(Statements::default()),
}));
// SAFETY: SQLite owns the boxed context after registration. Its function
// destructor disables the trace hook before freeing the shared context.
unsafe {
let result = ffi::sqlite3_create_function_v2(
db,
c"screenpipe_bulk".as_ptr(),
5,
ffi::SQLITE_UTF8,
context.cast(),
Some(value),
None,
None,
Some(destroy),
);
if result != ffi::SQLITE_OK {
return Err(storage_error("cannot register Parquet reader"));
}
if ffi::sqlite3_trace_v2(
db,
(ffi::SQLITE_TRACE_STMT | ffi::SQLITE_TRACE_PROFILE) as u32,
Some(trace),
context.cast(),
) != ffi::SQLITE_OK
{
return Err(storage_error("cannot install payload statement leases"));
}
}
}
if logical {
let query_only: i64 = sqlx::query_scalar("PRAGMA query_only")
.fetch_one(&mut *conn)
.await?;
sqlx::query("PRAGMA query_only=OFF")
.execute(&mut *conn)
.await?;
let result = async {
for table in TABLES.iter().filter(|t| t.name != "elements") {
// SQLite resolves the target when the view is queried, after
// bootstrap has created this generation's main-schema views.
sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
"CREATE TEMP VIEW {} AS SELECT * FROM main.{}",
table.name,
table.view()
)))
.execute(&mut *conn)
.await?;
}
Ok::<_, sqlx::Error>(())
}
.await;
if query_only != 0 {
sqlx::query("PRAGMA query_only=ON")
.execute(&mut *conn)
.await?;
}
result?;
}
Ok(())
}
pub(crate) async fn register_hash(conn: &mut SqliteConnection) -> Result<(), sqlx::Error> {
let mut locked = conn.lock_handle().await?;
// SAFETY: SQLx holds the live connection exclusively during registration.
let result = unsafe { register_hash_raw(locked.as_raw_handle().as_ptr()) };
if result != ffi::SQLITE_OK {
return Err(storage_error("cannot register payload identity function"));
}
Ok(())
}
/// Independent recovery connections evaluate the persistent payload-identity
/// index before storage pools exist, including on the first open after restart.
pub(crate) fn register_hash_extension() -> Result<(), sqlx::Error> {
unsafe extern "C" fn init(
db: *mut ffi::sqlite3,
_: *mut *mut std::ffi::c_char,
_: *const ffi::sqlite3_api_routines,
) -> c_int {
register_hash_raw(db)
}
// SAFETY: init has SQLite's extension ABI and retains no connection state.
let result = unsafe { ffi::sqlite3_auto_extension(Some(init)) };
if result != ffi::SQLITE_OK {
return Err(storage_error(format!(
"cannot register payload identity extension: SQLite error code {result}"
)));
}
Ok(())
}
unsafe fn register_hash_raw(db: *mut ffi::sqlite3) -> c_int {
// SAFETY: the callback borrows only SQLite's argument and copies its result.
unsafe {
ffi::sqlite3_create_function_v2(
db,
c"screenpipe_payload_sha256".as_ptr(),
1,
ffi::SQLITE_UTF8 | ffi::SQLITE_DETERMINISTIC | ffi::SQLITE_INNOCUOUS,
std::ptr::null_mut(),
Some(hash),
None,
None,
None,
)
}
}
unsafe extern "C" fn hash(
output: *mut ffi::sqlite3_context,
_: c_int,
args: *mut *mut ffi::sqlite3_value,
) {
use sha2::{Digest, Sha256};
let bytes = ffi::sqlite3_value_blob(*args);
let len = ffi::sqlite3_value_bytes(*args);
let bytes = if len == 0 {
&[]
} else if bytes.is_null() {
ffi::sqlite3_result_error_nomem(output);
return;
} else {
std::slice::from_raw_parts(bytes.cast::<u8>(), len as usize)
};
let digest = Sha256::digest(bytes);
ffi::sqlite3_result_blob(
output,
digest.as_ptr().cast(),
digest.len() as c_int,
ffi::SQLITE_TRANSIENT(),
);
}
unsafe extern "C" fn destroy(pointer: *mut c_void) {
let context = Box::from_raw(pointer.cast::<Context>());
ffi::sqlite3_trace_v2(
context.database as *mut ffi::sqlite3,
0,
None,
std::ptr::null_mut(),
);
drop(context);
}
unsafe extern "C" fn trace(
event: u32,
pointer: *mut c_void,
statement: *mut c_void,
_: *mut c_void,
) -> c_int {
let context = &*pointer.cast::<Context>();
// Pin before SQLite opens a snapshot, including nested statements during
// preparation. Admission is independent of file unlinking: new snapshots
// see committed current locators while older snapshots defer reclamation.
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let mut state = context.statements.lock().map_err(storage_error)?;
if event == ffi::SQLITE_TRACE_STMT as u32 {
if state.lease.is_none() {
state.lease = Some(StatementLease::new(&context.storage));
}
state.active.insert(statement as usize);
} else if event == ffi::SQLITE_TRACE_PROFILE as u32 {
state.active.remove(&(statement as usize));
if state.active.is_empty()
&& ffi::sqlite3_get_autocommit(context.database as *mut ffi::sqlite3) != 0
{
state.lease = None;
}
}
Ok::<_, sqlx::Error>(())
}));
if !matches!(result, Ok(Ok(()))) {
ffi::sqlite3_interrupt(context.database as *mut ffi::sqlite3);
}
0
}
unsafe extern "C" fn value(
output: *mut ffi::sqlite3_context,
count: c_int,
arguments: *mut *mut ffi::sqlite3_value,
) {
let context = &*ffi::sqlite3_user_data(output).cast::<Context>();
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
if count != 5 {
return Err(storage_error("invalid bulk reader arguments"));
}
let args = std::slice::from_raw_parts(arguments, 5);
let string = |index: usize| -> Result<&str, sqlx::Error> {
let text = ffi::sqlite3_value_text(args[index]);
if text.is_null() {
return Err(storage_error("missing bulk file location"));
}
CStr::from_ptr(text.cast()).to_str().map_err(storage_error)
};
let path = string(0)?;
let hash = string(1)?;
let table = usize::try_from(ffi::sqlite3_value_int64(args[2])).map_err(storage_error)?;
let id = ffi::sqlite3_value_int64(args[3]);
let column = usize::try_from(ffi::sqlite3_value_int64(args[4])).map_err(storage_error)?;
let rows = context.storage.bulk_records(path, hash, table)?;
let position = rows
.binary_search_by_key(&id, |r| r.id)
.map_err(|_| storage_error("bulk row is absent"))?;
let value = rows[position]
.values
.get(column)
.ok_or_else(|| storage_error("bulk column is absent"))?;
match value {
Value::Null => ffi::sqlite3_result_null(output),
Value::Text(s) => ffi::sqlite3_result_text64(
output,
s.as_ptr().cast(),
s.len() as u64,
ffi::SQLITE_TRANSIENT(),
ffi::SQLITE_UTF8 as u8,
),
Value::Integer(i) => ffi::sqlite3_result_int64(output, *i),
Value::Real(f) => ffi::sqlite3_result_double(output, *f),
}
Ok::<_, sqlx::Error>(())
}));
if !matches!(result, Ok(Ok(()))) {
// Captured strings and paths never enter SQLite diagnostics.
ffi::sqlite3_result_error(
output,
c"Parquet payload unavailable or invalid".as_ptr(),
-1,
);
}
}