Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

33.7 KB
Raw Blame
// screenpipe — AI that knows everything you've seen, said, or heard
// https://screenpipe.com

use super::{declaration, names, TABLE};
use crate::storage::{
    bulk::{Record, Value},
    storage_error, HybridStorage,
};
use rusqlite::{
    ffi, params, params_from_iter,
    types::{Value as SqlValue, ValueRef},
    vtab::*,
    Connection, OptionalExtension, Result,
};
use std::{
    collections::VecDeque,
    sync::{Arc, LazyLock},
};

pub(crate) async fn register(
    conn: &mut sqlx::SqliteConnection,
    storage: Arc<HybridStorage>,
) -> Result<(), sqlx::Error> {
    let mut handle = conn.lock_handle().await?;
    // SQLx owns the handle. The module borrows it only on its SQLite worker;
    // from_handle never closes the borrowed database.
    unsafe {
        Connection::from_handle(handle.as_raw_handle().as_ptr())
            .map_err(storage_error)?
            .create_module(
                c"screenpipe_elements",
                update_module_with_tx::<Elements>(),
                Some(storage),
            )
            .map_err(storage_error)?;
    }
    Ok(())
}

fn failure(message: &str) -> rusqlite::Error {
    rusqlite::Error::ModuleError(message.into())
}
fn constraint(message: &str) -> rusqlite::Error {
    rusqlite::Error::SqliteFailure(
        ffi::Error::new(ffi::SQLITE_CONSTRAINT),
        Some(message.into()),
    )
}

trait CachedSql {
    fn cached_execute<P: rusqlite::Params>(&self, sql: &str, params: P) -> Result<usize>;
    fn cached_query_row<T, P: rusqlite::Params, F: FnOnce(&rusqlite::Row<'_>) -> Result<T>>(
        &self,
        sql: &str,
        params: P,
        read: F,
    ) -> Result<T>;
}

impl CachedSql for Connection {
    fn cached_execute<P: rusqlite::Params>(&self, sql: &str, params: P) -> Result<usize> {
        self.prepare_cached(sql)?.execute(params)
    }
    fn cached_query_row<T, P: rusqlite::Params, F: FnOnce(&rusqlite::Row<'_>) -> Result<T>>(
        &self,
        sql: &str,
        params: P,
        read: F,
    ) -> Result<T> {
        self.prepare_cached(sql)?.query_row(params, read)
    }
}

pub(super) fn sql_value(value: &Value) -> SqlValue {
    match value {
        Value::Null => SqlValue::Null,
        Value::Integer(v) => SqlValue::Integer(*v),
        Value::Real(v) => SqlValue::Real(*v),
        Value::Text(v) => SqlValue::Text(v.clone()),
    }
}
fn value(value: SqlValue) -> Result<Value> {
    Ok(match value {
        SqlValue::Null => Value::Null,
        SqlValue::Integer(v) => Value::Integer(v),
        SqlValue::Real(v) => Value::Real(v),
        SqlValue::Text(v) => Value::Text(v),
        SqlValue::Blob(_) => return Err(constraint("element column type mismatch")),
    })
}
fn staged(row: &rusqlite::Row<'_>) -> Result<(Record, bool)> {
    let values = (0..TABLE.columns.len())
        .map(|i| value(row.get(i + 2)?))
        .collect::<Result<Vec<_>>>()?;
    Ok((
        Record {
            id: row.get(0)?,
            generation: row.get(1)?,
            values,
        },
        row.get::<_, i64>(TABLE.columns.len() + 2)? != 0,
    ))
}
struct RowSql {
    projection: String,
    lookup: String,
    upsert: String,
    resident: String,
}

static ROW_SQL: LazyLock<RowSql> = LazyLock::new(|| {
    let columns = names();
    let projection = format!("id,_archive_generation,{columns},_archive_deleted");
    // Preserve the tombstone's generation when an ID is deleted and reused.
    // An encoder holding the old generation must never accept the new row.
    let updates = format!("_archive_deleted,_archive_file,{columns}")
        .split(',')
        .map(|column| format!("{column}=excluded.{column}"))
        .collect::<Vec<_>>()
        .join(",");
    RowSql {
        lookup: format!("SELECT {projection} FROM _bulk_element_rows WHERE id=?"),
        upsert: format!("INSERT INTO _bulk_element_rows(id,_archive_generation,_archive_deleted,_archive_file,{columns}) VALUES({}) ON CONFLICT(id) DO UPDATE SET _archive_generation=max(_bulk_element_rows._archive_generation+1,excluded._archive_generation),{updates}", vec!["?";TABLE.columns.len()+4].join(",")),
        resident: format!("SELECT COALESCE((SELECT {} FROM _bulk_element_rows WHERE id=?),0)", TABLE.all_bytes("")),
        projection,
    }
});

fn location(db: &Connection, id: i64) -> Result<Option<(i64, String, String)>> {
    db.cached_query_row("SELECT r.file_id,p.path,p.checksum FROM _bulk_element_ranges r JOIN _bulk_files p ON p.id=r.file_id WHERE r.first_id=(SELECT max(first_id) FROM _bulk_element_ranges WHERE first_id<=?1) AND r.last_id>=?1", [id], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()
}
fn lookup(db: &Connection, storage: &HybridStorage, id: i64) -> Result<Option<Record>> {
    if let Some((row, deleted)) = db
        .cached_query_row(&ROW_SQL.lookup, [id], staged)
        .optional()?
    {
        return Ok((!deleted).then_some(row));
    }
    let Some((_, path, hash)) = location(db, id)? else {
        return Ok(None);
    };
    let index = storage
        .element_index(&path, &hash)
        .map_err(|_| failure("element index unavailable"))?;
    let Ok(position) = index.rows.binary_search_by_key(&id, |r| r.id) else {
        return Ok(None);
    };
    let rows = storage
        .bulk_frame_records(&path, &hash, index.rows[position].frame, &index)
        .map_err(|_| failure("element archive unavailable"))?;
    Ok(rows
        .binary_search_by_key(&id, |r| r.id)
        .ok()
        .map(|i| rows[i].clone()))
}

#[repr(C)]
struct Elements {
    base: ffi::sqlite3_vtab,
    database: Connection,
    storage: Arc<HybridStorage>,
}

unsafe impl<'v> VTab<'v> for Elements {
    type Aux = Arc<HybridStorage>;
    type Cursor = Cursor;
    fn connect(
        db: &mut VTabConnection,
        aux: Option<&Self::Aux>,
        _: &[&[u8]],
    ) -> Result<(String, Self)> {
        // Writes touch SQLite shadow state in the same transaction. SQLite
        // rolls these writes back with a failed outer statement/savepoint.
        let storage = aux
            .ok_or_else(|| failure("element storage missing"))?
            .clone();
        // SQLx owns the connection. Cached helper statements live for the
        // writer transaction and are finalized at its commit/rollback boundary.
        let database = unsafe { Connection::from_handle(db.handle())? };
        database.set_prepared_statement_cache_capacity(32);
        Ok((
            format!(
                "CREATE TABLE x(id INTEGER PRIMARY KEY,{},_archive_generation INTEGER HIDDEN)",
                declaration()
            ),
            Self {
                base: Default::default(),
                database,
                storage,
            },
        ))
    }
    fn best_index(&self, info: &mut IndexInfo) -> Result<()> {
        let mut constraints = Vec::new();
        let mut point = false;
        let mut frame = false;
        for (c, mut usage) in info.constraints_and_usages() {
            if !c.is_usable() {
                continue;
            }
            let column = c.column();
            let op = match c.operator() {
                IndexConstraintOp::SQLITE_INDEX_CONSTRAINT_EQ => "=",
                IndexConstraintOp::SQLITE_INDEX_CONSTRAINT_GT => ">",
                IndexConstraintOp::SQLITE_INDEX_CONSTRAINT_GE => ">=",
                IndexConstraintOp::SQLITE_INDEX_CONSTRAINT_LT => "<",
                IndexConstraintOp::SQLITE_INDEX_CONSTRAINT_LE => "<=",
                _ => continue,
            };
            if column == -1 || column == 0 || (column == 1 && op == "=") {
                point |= column <= 0 && op == "=";
                frame |= column == 1;
                constraints.push(format!("{}:{op}", if column == 1 { "frame" } else { "id" }));
                usage.set_argv_index(constraints.len() as i32);
            }
        }
        let ordering: Vec<_> = info
            .order_bys()
            .map(|o| (o.column(), o.is_order_by_desc()))
            .collect();
        let desc = ordering.len() == 1 && ordering[0].0 <= 0 && ordering[0].1;
        if ordering.len() == 1 && ordering[0].0 <= 0 {
            info.set_order_by_consumed(true);
        }
        info.set_idx_num(i32::from(desc));
        info.set_idx_str(&constraints.join(","));
        info.set_estimated_rows(if point {
            1
        } else if frame {
            200
        } else {
            20_000_000
        });
        info.set_estimated_cost(if point {
            10.0
        } else if frame {
            200.0
        } else {
            20_000_000.0
        });
        if point {
            info.set_idx_flags(IndexFlags::SQLITE_INDEX_SCAN_UNIQUE);
        }
        Ok(())
    }
    fn open(&mut self) -> Result<Cursor> {
        Ok(Cursor {
            base: Default::default(),
            db: unsafe { Connection::from_handle(self.database.handle())? },
            storage: self.storage.clone(),
            files_sql: String::new(),
            files_args: Vec::new(),
            files_after: None,
            files_done: false,
            archive: None,
            archive_location: None,
            archive_position: 0,
            archive_indices: Vec::new(),
            lower: i64::MIN,
            upper: i64::MAX,
            frame: None,
            pending: VecDeque::new(),
            pending_sql: String::new(),
            pending_args: Vec::new(),
            pending_seek_args: Vec::new(),
            pending_after: None,
            pending_done: false,
            descending: false,
            current: None,
        })
    }
}
impl CreateVTab<'_> for Elements {
    const KIND: VTabKind = VTabKind::Default;
}

impl Elements {
    fn write(&self, id: i64, mut row: Option<Record>, inserting: bool) -> Result<i64> {
        let db = &self.database;
        let old = lookup(&db, &self.storage, id)?;
        if inserting && old.is_some() {
            return Err(constraint("element ID already exists"));
        }
        if !inserting && old.is_none() {
            return Err(constraint("element ID is absent"));
        }
        let file = location(&db, id)?.map(|r| r.0);
        if let Some(new) = row.as_mut() {
            // Affinities are explicit for virtual tables; identity, required
            // fields, and relationships have the same accepted value types.
            for (i, column) in TABLE.columns.iter().enumerate() {
                use crate::storage::bulk::Kind;
                let current = std::mem::replace(&mut new.values[i], Value::Null);
                new.values[i] = match (column.kind, current) {
                    (_, Value::Null) => Value::Null,
                    (Kind::Text, value @ Value::Text(_))
                    | (Kind::Integer, value @ Value::Integer(_))
                    | (Kind::Real, value @ Value::Real(_)) => value,
                    (Kind::Text, Value::Integer(value)) => Value::Text(value.to_string()),
                    (Kind::Text, Value::Real(value)) => Value::Text(db.cached_query_row(
                        "SELECT CAST(? AS TEXT)",
                        [value],
                        |r| r.get(0),
                    )?),
                    (Kind::Real, Value::Integer(value)) => Value::Real(value as f64),
                    (Kind::Integer, Value::Text(value)) => Value::Integer(
                        value
                            .trim()
                            .parse()
                            .map_err(|_| constraint("element integer affinity mismatch"))?,
                    ),
                    (Kind::Real, Value::Text(value)) => Value::Real(
                        value
                            .trim()
                            .parse()
                            .map_err(|_| constraint("element real affinity mismatch"))?,
                    ),
                    (Kind::Integer, Value::Real(value))
                        if value.is_finite()
                            && value.fract() == 0.0
                            && value >= i64::MIN as f64
                            && value < -(i64::MIN as f64) =>
                    {
                        Value::Integer(value as i64)
                    }
                    _ => return Err(constraint("element column type mismatch")),
                };
            }
            for index in [0, 1, 2, 5, 11] {
                if new.values[index] == Value::Null {
                    return Err(constraint("required element column is NULL"));
                }
            }
            let Value::Integer(frame) = new.values[0] else {
                return Err(constraint("invalid element frame"));
            };
            if !db.cached_query_row(
                "SELECT EXISTS(SELECT 1 FROM frames WHERE id=?)",
                [frame],
                |r| r.get::<_, bool>(0),
            )? {
                return Err(constraint("element frame is absent"));
            }
            if let Value::Integer(parent) = new.values[4] {
                db.cached_execute(
                    "INSERT OR IGNORE INTO _bulk_element_checks VALUES(?)",
                    [parent],
                )?;
            }
            new.generation = old.as_ref().map_or(1, |r| r.generation + 1);
        }
        if row.is_none() {
            db.cached_execute("INSERT OR IGNORE INTO _bulk_element_checks VALUES(?)", [id])?;
        }
        let resident: i64 = if inserting {
            0
        } else {
            db.cached_query_row(&ROW_SQL.resident, [id], |r| r.get(0))?
        };
        // Frame references enforce the ordinary frame FK. Parent references
        // validate the complete transaction, including multi-row tree deletes.
        for (record, delta) in [(old.as_ref(), -1), (row.as_ref(), 1)] {
            if let Some(record) = record {
                let visibility = match record.values[13] {
                    Value::Integer(value) => value,
                    _ => 0,
                };
                let is_null = i64::from(record.values[13] == Value::Null);
                db.cached_execute(
                    "INSERT OR IGNORE INTO _bulk_element_kinds(source,role) VALUES(?,?)",
                    params![sql_value(&record.values[1]), sql_value(&record.values[2])],
                )?;
                let kind: i64 = db.cached_query_row(
                    "SELECT id FROM _bulk_element_kinds WHERE source=? AND role=?",
                    params![sql_value(&record.values[1]), sql_value(&record.values[2])],
                    |r| r.get(0),
                )?;
                db.cached_execute("INSERT INTO _bulk_element_groups VALUES(?1,?2,?3,?4,?5) ON CONFLICT(frame_id,kind_id,visibility,is_null) DO UPDATE SET rows=rows+excluded.rows",params![sql_value(&record.values[0]),kind,visibility,is_null,delta])?;
                if delta < 0 {
                    db.cached_execute("DELETE FROM _bulk_element_groups WHERE frame_id=?1 AND kind_id=?2 AND visibility=?3 AND is_null=?4 AND rows=0",params![sql_value(&record.values[0]),kind,visibility,is_null])?;
                }
                for (index, table, key) in [(4, "_bulk_element_parent_refs", "parent_id")] {
                    if let Value::Integer(value) = record.values[index] {
                        db.cached_execute(&format!("INSERT INTO {table}({key},rows) VALUES(?1,?2) ON CONFLICT({key}) DO UPDATE SET rows=rows+excluded.rows"),params![value,delta])?;
                        if delta < 0 {
                            db.cached_execute(
                                &format!("DELETE FROM {table} WHERE {key}=? AND rows=0"),
                                [value],
                            )?;
                        }
                    }
                }
            }
        }
        if let Some(old) = &old {
            if matches!(&old.values[3], Value::Text(text) if !text.is_empty()) {
                db.cached_execute("INSERT INTO elements_fts(elements_fts,rowid,text,role,frame_id) VALUES('delete',?,?,?,?)", params![id,sql_value(&old.values[3]),sql_value(&old.values[2]),sql_value(&old.values[0])])?;
            }
        }
        let new_bytes = row.as_ref().map_or(0, Record::bytes) as i64;
        let generation = row
            .as_ref()
            .map_or_else(|| old.as_ref().unwrap().generation + 1, |r| r.generation);
        let deleted = row.is_none();
        let mut args = vec![
            SqlValue::Integer(id),
            SqlValue::Integer(generation),
            SqlValue::Integer(i64::from(deleted)),
            file.map_or(SqlValue::Null, SqlValue::Integer),
        ];
        if let Some(row) = &row {
            args.extend(row.values.iter().map(sql_value));
        } else {
            args.resize(TABLE.columns.len() + 4, SqlValue::Null);
        }
        db.cached_execute(&ROW_SQL.upsert, params_from_iter(args))?;
        if let Some(row) = row {
            if matches!(&row.values[3],Value::Text(t) if !t.is_empty()) {
                db.cached_execute(
                    "INSERT INTO elements_fts(rowid,text,role,frame_id) VALUES(?,?,?,?)",
                    params![
                        id,
                        sql_value(&row.values[3]),
                        sql_value(&row.values[2]),
                        sql_value(&row.values[0])
                    ],
                )?;
            }
        }
        db.cached_execute(
            "UPDATE storage_metadata SET staging_bytes=staging_bytes-?1+?2,revision=revision+1",
            params![resident, new_bytes],
        )?;
        db.cached_execute("UPDATE _bulk_element_state SET version=version+1", [])?;
        if let Some(file) = file {
            db.cached_execute("UPDATE _bulk_files SET state='dirty' WHERE id=?", [file])?;
        }
        if inserting {
            db.cached_execute("INSERT INTO sqlite_sequence(name,seq) SELECT 'elements',0 WHERE NOT EXISTS(SELECT 1 FROM sqlite_sequence WHERE name='elements')",[])?;
            db.cached_execute(
                "UPDATE sqlite_sequence SET seq=max(seq,?) WHERE name='elements'",
                [id],
            )?;
        }
        Ok(id)
    }
    fn record(&self, args: &Values<'_>, id: i64) -> Result<Record> {
        Ok(Record {
            id,
            generation: 1,
            values: (0..TABLE.columns.len())
                .map(|i| value(args.get(i + 3)?))
                .collect::<Result<Vec<_>>>()?,
        })
    }
}
impl UpdateVTab<'_> for Elements {
    fn delete(&mut self, arg: ValueRef<'_>) -> Result<()> {
        self.write(arg.as_i64()?, None, false).map(|_| ())
    }
    fn insert(&mut self, args: &Inserts<'_>) -> Result<i64> {
        let db = &self.database;
        let explicit: Option<i64> = args.get(2)?;
        let rowid: Option<i64> = args.get(1)?;
        if explicit.is_some() && rowid.is_some() && explicit != rowid {
            return Err(constraint("element rowid and ID differ"));
        }
        let id = if let Some(id) = explicit.or(rowid) {
            id
        } else {
            let seq: i64 = db.cached_query_row(
                "SELECT COALESCE((SELECT seq FROM sqlite_sequence WHERE name='elements'),0)",
                [],
                |r| r.get(0),
            )?;
            seq.checked_add(1)
                .ok_or_else(|| constraint("element IDs exhausted"))?
        };
        let mut row = self.record(args, id)?;
        for index in [5, 11] {
            if row.values[index] == Value::Null {
                row.values[index] = Value::Integer(0);
            }
        }
        self.write(id, Some(row), true)
    }
    fn update(&mut self, args: &Updates<'_>) -> Result<()> {
        let id: i64 = args.get(0)?;
        if args.get::<i64>(1)? != id || args.get::<i64>(2)? != id {
            return Err(constraint("element IDs are immutable"));
        }
        self.write(id, Some(self.record(args, id)?), false)
            .map(|_| ())
    }
}
impl TransactionVTab<'_> for Elements {
    fn sync(&mut self) -> Result<()> {
        let result = (|| {
            let db = &self.database;
            {
                let mut checks = db.prepare("SELECT c.parent_id FROM _bulk_element_checks c JOIN _bulk_element_parent_refs p ON p.parent_id=c.parent_id WHERE p.rows>0")?;
                let mut rows = checks.query([])?;
                while let Some(row) = rows.next()? {
                    if lookup(&db, &self.storage, row.get(0)?)?.is_none() {
                        return Err(constraint("element parent is absent"));
                    }
                }
            }
            db.cached_execute("DELETE FROM _bulk_element_checks", [])?;
            Ok(())
        })();
        self.database.flush_prepared_statement_cache();
        result
    }
    fn commit(&mut self) -> Result<()> {
        self.database.flush_prepared_statement_cache();
        Ok(())
    }
    fn rollback(&mut self) -> Result<()> {
        self.database.flush_prepared_statement_cache();
        Ok(())
    }
}

#[repr(C)]
struct Cursor {
    base: ffi::sqlite3_vtab_cursor,
    db: Connection,
    storage: Arc<HybridStorage>,
    files_sql: String,
    files_args: Vec<SqlValue>,
    files_after: Option<i64>,
    files_done: bool,
    archive: Option<Arc<crate::storage::bulk::element_index::ElementIndex>>,
    archive_location: Option<(String, String)>,
    archive_position: usize,
    archive_indices: Vec<usize>,
    lower: i64,
    upper: i64,
    frame: Option<i64>,
    pending: VecDeque<(Record, bool)>,
    pending_sql: String,
    pending_args: Vec<SqlValue>,
    pending_seek_args: Vec<usize>,
    pending_after: Option<i64>,
    pending_done: bool,
    descending: bool,
    current: Option<CursorRow>,
}

enum CursorRow {
    Staged(Record),
    Archived(usize),
}
impl Cursor {
    fn current_id(&self) -> Result<i64> {
        match self.current.as_ref() {
            Some(CursorRow::Staged(row)) => Ok(row.id),
            Some(CursorRow::Archived(index)) => Ok(self.archive.as_ref().unwrap().rows[*index].id),
            None => Err(failure("element cursor exhausted")),
        }
    }
    fn fill_pending(&mut self) -> Result<()> {
        if !self.pending.is_empty() || self.pending_done {
            return Ok(());
        }
        let after = self.pending_after.map_or(String::new(), |_| {
            format!(" AND id{}?", if self.descending { "<" } else { ">" })
        });
        let mut args = self.pending_args.clone();
        if let Some(id) = self.pending_after {
            // This row already satisfied every bound in the scan direction.
            // Advance those bounds too: SQLite may seek using an original
            // >=/<= parameter instead of the appended strict cursor predicate,
            // otherwise rescanning the growing prefix on every 128-row page.
            for &index in &self.pending_seek_args {
                args[index] = SqlValue::Integer(id);
            }
            args.push(SqlValue::Integer(id));
        }
        let sql = format!(
            "SELECT {} FROM _bulk_element_rows WHERE {}{after} ORDER BY id {} LIMIT 128",
            ROW_SQL.projection,
            self.pending_sql,
            if self.descending { "DESC" } else { "ASC" }
        );
        let mut stmt = self.db.prepare_cached(&sql)?;
        let mut rows = stmt.query(params_from_iter(args))?;
        let mut bytes = 0;
        while let Some(row) = rows.next()? {
            let row = staged(row)?;
            let size = row.0.bytes();
            if !self.pending.is_empty() && bytes + size > self.storage.descriptor.budget.file_bytes
            {
                break;
            }
            bytes += size;
            self.pending_after = Some(row.0.id);
            self.pending.push_back(row);
        }
        if self.pending.is_empty() {
            self.pending_done = true;
        }
        Ok(())
    }
    fn fill_archive(&mut self) -> Result<()> {
        if self
            .archive
            .as_ref()
            .is_some_and(|_| self.archive_position < self.archive_indices.len())
        {
            return Ok(());
        }
        self.archive = None;
        while !self.files_done {
            let mut args = self.files_args.clone();
            let after = if let Some(id) = self.files_after {
                args.push(SqlValue::Integer(id));
                format!(
                    " AND r.first_id{}?",
                    if self.descending { "<" } else { ">" }
                )
            } else {
                String::new()
            };
            let sql = format!(
                "{}{after} ORDER BY r.first_id {} LIMIT 1",
                self.files_sql,
                if self.descending { "DESC" } else { "ASC" }
            );
            let file: Option<(i64, String, String)> = self
                .db
                .prepare_cached(&sql)?
                .query_row(params_from_iter(args), |r| {
                    Ok((r.get(0)?, r.get(1)?, r.get(2)?))
                })
                .optional()?;
            let Some((first_id, path, hash)) = file else {
                self.files_done = true;
                break;
            };
            self.files_after = Some(first_id);
            let rows = self
                .storage
                .element_index(&path, &hash)
                .map_err(|_| failure("element archive unavailable"))?;
            self.archive_position = 0;
            let begin = rows.rows.partition_point(|r| r.id < self.lower);
            let end = rows.rows.partition_point(|r| r.id <= self.upper);
            self.archive_indices = if let Some(frame) = self.frame {
                let frames = self
                    .storage
                    .element_frame_rows(format!("{path}:{hash}"), &rows.rows)
                    .map_err(|_| failure("element frame index unavailable"))?;
                frames
                    .get(&frame)
                    .map(|ids| {
                        ids.iter()
                            .copied()
                            .filter(|i| *i >= begin && *i < end)
                            .collect()
                    })
                    .unwrap_or_default()
            } else {
                (begin..end).collect()
            };
            if self.descending {
                self.archive_indices.reverse();
            }
            if !self.archive_indices.is_empty() {
                self.archive = Some(rows);
                self.archive_location = Some((path, hash));
                break;
            }
        }
        Ok(())
    }
    fn advance(&mut self) -> Result<()> {
        self.current = None;
        loop {
            self.fill_pending()?;
            self.fill_archive()?;
            let archived = self
                .archive
                .as_ref()
                .map(|r| &r.rows[self.archive_indices[self.archive_position]]);
            let pending = self.pending.front();
            let take_pending = match (pending, archived) {
                (None, None) => return Ok(()),
                (Some(_), None) => true,
                (None, Some(_)) => false,
                (Some((p, _)), Some(a)) => {
                    if self.descending {
                        p.id >= a.id
                    } else {
                        p.id <= a.id
                    }
                }
            };
            if take_pending {
                let (row, deleted) = self.pending.pop_front().unwrap();
                if archived.is_some_and(|a| a.id == row.id) {
                    self.archive_position += 1;
                }
                if deleted {
                    continue;
                }
                self.current = Some(CursorRow::Staged(row));
            } else {
                self.current = Some(CursorRow::Archived(
                    self.archive_indices[self.archive_position],
                ));
                self.archive_position += 1;
            }
            return Ok(());
        }
    }
}
unsafe impl VTabCursor for Cursor {
    fn filter(&mut self, idx: i32, description: Option<&str>, args: &Filters<'_>) -> Result<()> {
        self.descending = idx != 0;
        self.lower = i64::MIN;
        self.upper = i64::MAX;
        self.frame = None;
        self.pending.clear();
        self.pending_after = None;
        self.pending_done = false;
        self.pending_seek_args.clear();
        self.files_after = None;
        self.files_done = false;
        self.archive = None;
        self.archive_location = None;
        self.current = None;
        let mut range = Vec::new();
        let mut pending = Vec::new();
        let mut range_args = Vec::new();
        let mut pending_args = Vec::new();
        let mut frame = None;
        for (i, item) in description
            .unwrap_or("")
            .split(',')
            .filter(|s| !s.is_empty())
            .enumerate()
        {
            let (column, op) = item
                .split_once(':')
                .ok_or_else(|| failure("invalid element query plan"))?;
            let value: SqlValue = args.get(i)?;
            if column == "frame" {
                if let SqlValue::Integer(id) = &value {
                    self.frame = Some(*id);
                }
                frame = Some(value);
                continue;
            }
            if let SqlValue::Integer(id) = &value {
                match op {
                    "=" => {
                        self.lower = self.lower.max(*id);
                        self.upper = self.upper.min(*id);
                    }
                    ">" => self.lower = self.lower.max(id.saturating_add(1)),
                    ">=" => self.lower = self.lower.max(*id),
                    "<" => self.upper = self.upper.min(id.saturating_sub(1)),
                    "<=" => self.upper = self.upper.min(*id),
                    _ => (),
                }
            }
            let condition = match op {
                "=" => "r.first_id=(SELECT max(first_id) FROM _bulk_element_ranges WHERE first_id<=?) AND r.last_id>=?",
                ">" | ">=" => "r.first_id>=COALESCE((SELECT max(first_id) FROM _bulk_element_ranges WHERE first_id<=?),?) AND r.last_id>=?",
                "<" | "<=" => "r.first_id<=?",
                _ => return Err(failure("invalid element range")),
            };
            range.push(condition.to_owned());
            range_args.push(value.clone());
            if op == "=" {
                range_args.push(value.clone());
            } else if matches!(op, ">" | ">=") {
                range_args.extend([value.clone(), value.clone()]);
            }
            if matches!(
                (self.descending, op),
                (false, ">" | ">=") | (true, "<" | "<=")
            ) {
                self.pending_seek_args.push(pending_args.len());
            }
            pending.push(format!("id{op}?"));
            pending_args.push(value);
        }
        if let Some(frame) = &frame {
            range.push("f.frame_id=?".into());
            range_args.push(frame.clone());
        }
        let where_sql = if range.is_empty() {
            "1".into()
        } else {
            range.join(" AND ")
        };
        let frame_join = if frame.is_some() {
            "JOIN _bulk_element_frames f ON f.file_id=r.file_id"
        } else {
            ""
        };
        self.files_sql=format!("SELECT r.first_id,p.path,p.checksum FROM _bulk_element_ranges r JOIN _bulk_files p ON p.id=r.file_id {frame_join} WHERE {where_sql}");
        self.files_args = range_args;
        if let Some(frame) = frame {
            pending.push("(frame_id=? OR _archive_file IN (SELECT file_id FROM _bulk_element_frames WHERE frame_id=?))".into());
            pending_args.extend([frame.clone(), frame]);
        }
        self.pending_sql = if pending.is_empty() {
            "1".into()
        } else {
            pending.join(" AND ")
        };
        self.pending_args = pending_args;
        self.advance()
    }
    fn next(&mut self) -> Result<()> {
        self.advance()
    }
    fn eof(&self) -> bool {
        self.current.is_none()
    }
    fn column(&self, ctx: &mut Context, index: i32) -> Result<()> {
        let current = self
            .current
            .as_ref()
            .ok_or_else(|| failure("element cursor exhausted"))?;
        let full;
        let row = match current {
            CursorRow::Staged(row) => row,
            CursorRow::Archived(position) => {
                let archive = self.archive.as_ref().unwrap();
                let row = &archive.rows[*position];
                match index {
                    0 => return ctx.set_result(&row.id),
                    1 => return ctx.set_result(&row.frame),
                    2 => return ctx.set_result(&archive.sources[row.source as usize]),
                    3 => return ctx.set_result(&archive.roles[row.role as usize]),
                    12 => return ctx.set_result(&row.sort),
                    14 => return ctx.set_result(&(row.present & 1 != 0).then_some(row.visible)),
                    15 => return ctx.set_result(&(row.present & 2 != 0).then_some(row.redacted)),
                    16 => return ctx.set_result(&row.generation),
                    _ => (),
                }
                let (path, hash) = self.archive_location.as_ref().unwrap();
                // A history scan visits many frames in the same file. Decode
                // that bounded file once instead of decompressing its pages
                // again for each frame. Point/frame reads keep their selective
                // projection; metadata-only scans still use the index above.
                full = if self.frame.is_some() || self.lower == self.upper {
                    self.storage
                        .bulk_frame_records(path, hash, row.frame, archive)
                } else {
                    self.storage.bulk_records(path, hash, 0)
                }
                .map_err(|_| failure("element archive unavailable"))?;
                let position = full
                    .binary_search_by_key(&row.id, |r| r.id)
                    .map_err(|_| failure("element row is absent"))?;
                &full[position]
            }
        };
        if index == 0 {
            ctx.set_result(&row.id)
        } else if index as usize == TABLE.columns.len() + 1 {
            ctx.set_result(&row.generation)
        } else {
            ctx.set_result(&sql_value(&row.values[index as usize - 1]))
        }
    }
    fn rowid(&self) -> Result<i64> {
        self.current_id()
    }
}