YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
// screenpipe — AI that knows everything you've seen, said, or heard
// https://screenpipe.com
//! One bounded SQLite read transaction per logical hybrid read. Immutable
//! Parquet generations remain pinned until the read and response admission end.
use crate::cancellable_query::CancellableReadConnection;
use sqlx::{pool::PoolConnection, Sqlite, SqliteConnection, SqlitePool, Transaction};
use std::{
ops::{Deref, DerefMut},
sync::Arc,
time::{Duration, Instant},
};
use tokio::sync::{Mutex, OwnedMutexGuard, OwnedRwLockReadGuard};
use tokio_util::sync::CancellationToken;
type SharedConnection = Arc<Mutex<Option<CancellableReadConnection>>>;
struct Context {
pool: SqlitePool,
connection: SharedConnection,
revision: tokio::sync::OnceCell<(i64, i64)>,
lease: Arc<OwnedRwLockReadGuard<()>>,
}
tokio::task_local! { static SNAPSHOT: Arc<Context>; }
/// Statements within one read borrow its connection serially. Other readers
/// and the existing writer continue on their own SQLite connections.
pub enum ReadConnection {
Pool(PoolConnection<Sqlite>),
Cancellable(CancellableReadConnection),
Transaction(Transaction<'static, Sqlite>),
Snapshot(OwnedMutexGuard<Option<CancellableReadConnection>>),
}
impl Deref for ReadConnection {
type Target = SqliteConnection;
fn deref(&self) -> &SqliteConnection {
match self {
Self::Pool(c) => c,
Self::Cancellable(c) => c,
Self::Transaction(c) => c,
Self::Snapshot(c) => c.as_ref().unwrap(),
}
}
}
impl DerefMut for ReadConnection {
fn deref_mut(&mut self) -> &mut SqliteConnection {
match self {
Self::Pool(c) => c,
Self::Cancellable(c) => c,
Self::Transaction(c) => c,
Self::Snapshot(c) => c.as_mut().unwrap(),
}
}
}
impl ReadConnection {
pub(super) async fn commit(self) -> Result<(), sqlx::Error> {
if let Self::Transaction(tx) = self {
tx.commit().await?;
}
Ok(())
}
}
fn context(pool: &SqlitePool) -> Option<Arc<Context>> {
SNAPSHOT.try_with(Arc::clone).ok().filter(|ctx| {
// Pool clones share their options address; another manager, including
// another in-memory database, must never borrow this transaction.
std::ptr::eq(ctx.pool.options(), pool.options())
})
}
pub(crate) fn in_snapshot(pool: &SqlitePool) -> bool {
context(pool).is_some()
}
pub(super) fn lease(pool: &SqlitePool) -> Option<Arc<OwnedRwLockReadGuard<()>>> {
context(pool).map(|ctx| Arc::clone(&ctx.lease))
}
pub(crate) async fn revision(pool: &SqlitePool) -> Result<(i64, i64), sqlx::Error> {
let ctx = context(pool).expect("snapshot belongs to this pool");
let revision = ctx.revision.get_or_try_init(|| async {
sqlx::query_as("SELECT revision,(SELECT revision FROM _storage_revocation WHERE id=1) FROM storage_metadata WHERE singleton=1")
.fetch_one(&mut *acquire(pool).await?).await
}).await?;
Ok(*revision)
}
async fn scoped(pool: &SqlitePool) -> Option<ReadConnection> {
let ctx = context(pool)?;
Some(ReadConnection::Snapshot(
Arc::clone(&ctx.connection).lock_owned().await,
))
}
pub(crate) async fn acquire(pool: &SqlitePool) -> Result<ReadConnection, sqlx::Error> {
if let Some(c) = scoped(pool).await {
return Ok(c);
}
Ok(ReadConnection::Pool(pool.acquire().await?))
}
pub(crate) async fn search(pool: &SqlitePool) -> Result<ReadConnection, sqlx::Error> {
if let Some(c) = scoped(pool).await {
return Ok(c);
}
Ok(ReadConnection::Cancellable(
CancellableReadConnection::acquire(
pool,
Instant::now() + crate::cancellable_query::SEARCH_QUERY_TIMEOUT,
CancellationToken::new(),
)
.await?,
))
}
pub(super) async fn frame_snapshot(pool: &SqlitePool) -> Result<ReadConnection, sqlx::Error> {
if let Some(c) = scoped(pool).await {
return Ok(c);
}
Ok(ReadConnection::Transaction(pool.begin().await?))
}
struct RequestSnapshot {
context: Arc<Context>,
lane: Option<tokio::sync::OwnedSemaphorePermit>,
cancellation: CancellationToken,
}
impl Drop for RequestSnapshot {
fn drop(&mut self) {
self.cancellation.cancel();
let context = Arc::clone(&self.context);
let lane = self.lane.take();
tokio::spawn(async move {
let _lane = lane;
if let Some(mut c) = context.connection.lock().await.take() {
if sqlx::query("ROLLBACK").execute(&mut *c).await.is_err() {
c.discard().await;
} else {
let _ = c.release().await;
}
}
});
}
}
pub(super) async fn run<T>(
storage: &Arc<super::HybridStorage>,
pool: &SqlitePool,
read: impl std::future::Future<Output = T>,
) -> Result<T, sqlx::Error> {
let lanes = storage.read_lanes.get_or_init(|| {
Arc::new(tokio::sync::Semaphore::new(
pool.options()
.get_max_connections()
.saturating_sub(1)
.max(1) as usize,
))
});
let lane = tokio::select! {
biased;
_ = storage.closing.cancelled() => return Err(sqlx::Error::PoolClosed),
lane = Arc::clone(lanes).acquire_owned() => lane.map_err(|_| sqlx::Error::PoolClosed)?,
};
// A cleanup backlog must not repeatedly revoke a page before it reaches
// final admission. Tokio's fair lock lets queued deletion and subsequent
// readers each progress. Recording uses neither side of this lock.
let _history = tokio::select! {
biased;
_ = storage.closing.cancelled() => return Err(sqlx::Error::PoolClosed),
history = storage.history.read() => history,
};
let lease = tokio::select! {
biased;
_ = storage.closing.cancelled() => return Err(sqlx::Error::PoolClosed),
lease = Arc::clone(&storage.leases).read_owned() => Arc::new(lease),
};
let cancellation = storage.closing.child_token();
let mut connection = CancellableReadConnection::acquire(
pool,
Instant::now() + Duration::from_secs(storage.descriptor.budget.operation_timeout_secs),
cancellation.clone(),
)
.await?;
sqlx::query("BEGIN").execute(&mut *connection).await?;
let owner = RequestSnapshot {
context: Arc::new(Context {
pool: pool.clone(),
connection: Arc::new(Mutex::new(Some(connection))),
revision: tokio::sync::OnceCell::new(),
lease,
}),
lane: Some(lane),
cancellation,
};
let (token, result) = SNAPSHOT
.scope(Arc::clone(&owner.context), async {
let token = storage.read_token(pool).await?;
Ok::<_, sqlx::Error>((token, read.await))
})
.await?;
// Finish the transaction before final admission, keeping the file pin.
// Dropping this future interrupts SQLite and rolls back before pool reuse.
if let Some(mut connection) = owner.context.connection.lock().await.take() {
if let Err(error) = sqlx::query("ROLLBACK").execute(&mut *connection).await {
connection.discard().await;
return Err(error);
}
connection.release().await?;
}
let _admission = token.admit(pool).await?;
Ok(result)
}
impl crate::DatabaseManager {
/// Share a SQLite snapshot across candidate selection, counts and payload
/// hydration. Ordinary writes do not revoke it; deletion and privacy do.
pub async fn read_snapshot<T>(
&self,
read: impl std::future::Future<Output = T>,
) -> Result<T, sqlx::Error> {
// Query futures contain nested search fan-out and hydration state.
// Keep that state on the heap across snapshot scopes so native Tokio
// workers retain their normal stack size.
let read = Box::pin(read);
let Some(storage) = &self.storage else {
return Ok(read.await);
};
if in_snapshot(&self.pool) {
return Ok(read.await);
}
tokio::time::timeout(
Duration::from_secs(storage.descriptor.budget.operation_timeout_secs),
run(storage, &self.pool, read),
)
.await
.map_err(|_| super::storage_error("payload read deadline exceeded"))?
}
pub(crate) async fn acquire_read(&self) -> Result<ReadConnection, sqlx::Error> {
acquire(&self.pool).await
}
}