:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(deltalite): add a GIL-free restore and a delta-rs table check (#119677)
Daniel RCafdd0d1f267a
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 92d790a200c8..e96ef7757838 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -3982,7 +3982,7 @@ dependencies = [[[package]]name = "deltalite-python"-version = "0.1.12"+version = "0.1.13"dependencies = ["arrow","arrow-array",diff --git a/rust/deltalite/core/src/handle.rs b/rust/deltalite/core/src/handle.rsindex 42745698c692..ded4753160bd 100644--- a/rust/deltalite/core/src/handle.rs+++ b/rust/deltalite/core/src/handle.rs@@ -23,9 +23,12 @@ use std::time::{Duration, Instant};use arrow_array::RecordBatch;use arrow_schema::SchemaRef;+use deltalake::kernel::transaction::CommitProperties;+use deltalake::operations::restore::RestoreMetrics;use deltalake::table::config::TablePropertiesExt as _;use deltalake::DeltaTable;use metrics::counter;+use serde_json::Value;use crate::errors::{Error, Result};use crate::limits::{env_switch, switch_setting};@@ -434,6 +437,42 @@ impl TableHandle {Ok(stats)}+ /// Fails when a vacuum deleted a file of `version`.+ pub async fn restore(+ &mut self,+ version: u64,+ commit_metadata: Option<HashMap<String, Value>>,+ ) -> Result<RestoreMetrics> {+ let result = self.restore_refreshed(version, commit_metadata).await;+ self.prefetch.clear();+ result+ }++ async fn restore_refreshed(+ &mut self,+ version: u64,+ commit_metadata: Option<HashMap<String, Value>>,+ ) -> Result<RestoreMetrics> {+ self.refresh().await?;+ let mut builder = self+ .table+ .clone()+ .restore()+ .with_version_to_restore(version);+ if let Some(metadata) = commit_metadata {+ builder =+ builder.with_commit_properties(CommitProperties::default().with_metadata(metadata));+ }+ let (table, metrics) = builder.await?;+ if self.adopt_commit_snapshot {+ self.table.state = table.state;+ self.list_on_next_refresh = self.on_checkpoint_boundary();+ } else {+ self.refresh().await?;+ }+ Ok(metrics)+ }+/// The open cost, handed out once so per-upsert stats sum to the handle's total.fn take_initial_open_ms(&mut self) -> u64 {if self.initial_open_reported {diff --git a/rust/deltalite/core/src/table.rs b/rust/deltalite/core/src/table.rsindex c4bb28f280a9..1cc4258f79e5 100644--- a/rust/deltalite/core/src/table.rs+++ b/rust/deltalite/core/src/table.rs@@ -59,6 +59,19 @@ pub async fn open_table(uri: &str, storage_options: HashMap<String, String>) ->Ok(table)}+/// `deltalake.DeltaTable.is_deltatable`'s check. It loads nothing, so a table with a damaged+/// log still counts as present.+pub async fn table_exists(uri: &str, storage_options: HashMap<String, String>) -> Result<bool> {+ let Ok(url) = deltalake::ensure_table_uri(uri) else {+ return Ok(false);+ };+ let Ok(builder) = DeltaTableBuilder::from_url(url) else {+ return Ok(false);+ };+ let table = builder.with_storage_options(storage_options).build()?;+ Ok(table.verify_deltatable_existence().await?)+}+/// [`open_table`] that also hands back the checkpoint cache the table's stores serve/// from, still holding the checkpoint bytes read by this load. The caller clears it once/// it is done and can clear it again after each later load through the same table.diff --git a/rust/deltalite/python/Cargo.toml b/rust/deltalite/python/Cargo.tomlindex 2dc94b04c601..54ba588631d4 100644--- a/rust/deltalite/python/Cargo.toml+++ b/rust/deltalite/python/Cargo.toml@@ -1,7 +1,7 @@[package]name = "deltalite-python"# Mirrors pyproject.toml in this directory (always identical); bump both together.-version = "0.1.12"+version = "0.1.13"edition = "2021"# CI compiles with this exact toolchain; cargo's MSRV-aware resolver uses it to# avoid picking dependency versions CI cannot build (aws-* crates move MSRV often).diff --git a/rust/deltalite/python/README.md b/rust/deltalite/python/README.mdindex 08e5d1e9ea46..034b3da9fcbb 100644--- a/rust/deltalite/python/README.md+++ b/rust/deltalite/python/README.md@@ -100,8 +100,10 @@ rejected (raising `DeltaLiteError`) rather than silently double-inserted.|---|---|| `DeltaLiteTable.open(uri, storage_options=None)` | Open an existing Delta table. `storage_options` is the usual object-store dict (S3/GCS/Azure/local). || `DeltaLiteTable.is_deltatable(uri, storage_options=None)` | `True` if a Delta table exists at `uri`. |+| `DeltaLiteTable.table_exists(uri, storage_options=None)` | `True` if the log at `uri` holds a commit or a checkpoint, the same check as `DeltaTable.is_deltatable`. Storage errors raise; they never read as `False`. || `.upsert(data, primary_keys, partition_key=None, **opts)` | Insert-or-replace `data` by key. Returns `UpsertStats`. See knobs below. || `.compact(**opts)` | Small-file compaction, the replacement for `DeltaTable.optimize.compact`. Returns a stats dict. See "Compaction" below. |+| `.restore(version, *, commit_metadata=None)` | delta-rs's restore to `version`, in one commit, with the GIL released. Returns `numRemovedFile`, `numRestoredFile` and the committed `version`. The handle moves onto the restored table. || `.version()` | Current table version (`int`). || `.reload()` | Bring the table state up to date with the log (incremental; falls back to a full re-open). || `.table_id()` | The table id from the metadata action (`str`). |diff --git a/rust/deltalite/python/pyproject.toml b/rust/deltalite/python/pyproject.tomlindex d8ce1097b214..f1102f1387af 100644--- a/rust/deltalite/python/pyproject.toml+++ b/rust/deltalite/python/pyproject.toml@@ -10,7 +10,7 @@ build-backend = "maturin"[project]name = "deltalite"# Mirrors `rust/deltalite/python/Cargo.toml` (always identical); bump both together.-version = "0.1.12"+version = "0.1.13"description = "Streaming partition upsert for Delta tables, replacing delta-rs SQL MERGE"readme = "README.md"requires-python = ">=3.12"diff --git a/rust/deltalite/python/src/lib.rs b/rust/deltalite/python/src/lib.rsindex d5bbdea8bac5..b7a440eedd6d 100644--- a/rust/deltalite/python/src/lib.rs+++ b/rust/deltalite/python/src/lib.rs@@ -16,7 +16,7 @@ use deltalite_core::errors::Error;use deltalite_core::handle::TableHandle;use deltalite_core::limits::ProcessLimits;use deltalite_core::schema::import_column;-use deltalite_core::table::{open_table, MultipartConfig};+use deltalite_core::table::{open_table, table_exists, MultipartConfig};use deltalite_core::upsert::{PruneStrategy, UpsertOptions};use pyo3::create_exception;use pyo3::exceptions::PyException;@@ -268,6 +268,20 @@ impl DeltaLiteTable {.is_ok())}+ /// `deltalake.DeltaTable.is_deltatable`'s check. Unlike `is_deltatable`, storage errors+ /// raise: a caller that reads "no table" can create a new table over an existing one.+ #[staticmethod]+ #[pyo3(signature = (uri, storage_options = None))]+ fn table_exists(+ py: Python<'_>,+ uri: String,+ storage_options: Option<HashMap<String, String>>,+ ) -> PyResult<bool> {+ let so = storage_options.unwrap_or_default();+ py.detach(|| runtime().block_on(table_exists(&uri, so)))+ .map_err(to_py_err)+ }+/// The table version this handle currently observes (-1 before any load).fn version(&self) -> i64 {self.handle.version()@@ -594,6 +608,30 @@ impl DeltaLiteTable {Ok(d.into_any().unbind())}+ /// Returns the keys delta-rs's `DeltaTable.restore` returns, plus the committed `version`.+ #[pyo3(signature = (version, *, commit_metadata = None))]+ fn restore(+ &mut self,+ py: Python<'_>,+ version: u64,+ commit_metadata: Option<HashMap<String, String>>,+ ) -> PyResult<Py<PyAny>> {+ let metadata = commit_metadata.map(|m| {+ m.into_iter()+ .map(|(k, v)| (k, Value::String(v)))+ .collect::<HashMap<String, Value>>()+ });+ let handle = &mut self.handle;+ let metrics = py+ .detach(|| runtime().block_on(handle.restore(version, metadata)))+ .map_err(to_py_err)?;+ let d = PyDict::new(py);+ d.set_item("numRemovedFile", metrics.num_removed_file)?;+ d.set_item("numRestoredFile", metrics.num_restored_file)?;+ d.set_item("version", self.handle.version())?;+ Ok(d.into_any().unbind())+ }+/// Commit metadata of the most recent `limit` commits, oldest first.fn history(&self, py: Python<'_>, limit: usize) -> PyResult<Py<PyAny>> {let table = self.handle.table().clone();diff --git a/rust/deltalite/python/tests/test_operations.py b/rust/deltalite/python/tests/test_operations.pyindex 885edeed70a4..f9fd85619b4b 100644--- a/rust/deltalite/python/tests/test_operations.py+++ b/rust/deltalite/python/tests/test_operations.py@@ -11,6 +11,7 @@ from __future__ import annotationsimport sysimport json+import shutilfrom pathlib import Pathimport pytest@@ -35,7 +36,7 @@ def simple(ids, part, v, extra="ABSENT"):return pa.table(d)-def content(uri) -> dict[tuple[str | None, str], int]:+def content(uri) -> dict[tuple[str | None, str | None], int | None]:"""{(id, partition): v} via the Python package -- also proves the log is readable."""tbl = deltalake.DeltaTable(uri).to_pyarrow_table()return dict(@@ -971,3 +972,79 @@ def test_upsert_creates_checkpoint_at_interval_boundary(tmp_path):# An off-boundary commit must not add another checkpoint.upsert_path(uri, simple(["c"], "p", 3), ["id"], PARTITION_KEY)assert list(log.glob("*.checkpoint.parquet")) == checkpoints+++def live_paths(uri) -> list[str]:+ root = Path(uri).resolve()+ return sorted(str(Path(u).relative_to(root)) for u in deltalake.DeltaTable(uri).file_uris())+++def test_restore_matches_delta_rs_restore(tmp_path):+ import deltalite++ p = "2026-07-23"+ base = tmp_path / "base"+ create_table(str(base), simple(["a", "b"], p, 1), True)+ upsert_path(str(base), simple(["c"], p, 2), ["id"], PARTITION_KEY)+ target = deltalake.DeltaTable(str(base)).version()+ expected = content(str(base))+ upsert_path(str(base), simple(["a", "d"], p, 3), ["id"], PARTITION_KEY)+ deltalake.DeltaTable(str(base)).optimize.compact()+ py_uri, lite_uri = str(tmp_path / "py"), str(tmp_path / "lite")+ shutil.copytree(base, py_uri)+ shutil.copytree(base, lite_uri)+ metadata = {"rolled_back_run_uuid": "run-1"}++ py_metrics = deltalake.DeltaTable(py_uri).restore(+ target, commit_properties=deltalake.CommitProperties(custom_metadata=metadata)+ )+ lite = deltalite.DeltaLiteTable.open(lite_uri)+ lite_metrics = lite.restore(target, commit_metadata=metadata)++ assert py_metrics["numRestoredFile"] > 0+ assert lite_metrics["numRemovedFile"] == py_metrics["numRemovedFile"]+ assert lite_metrics["numRestoredFile"] == py_metrics["numRestoredFile"]+ assert lite_metrics["version"] == deltalake.DeltaTable(py_uri).version()+ assert live_paths(lite_uri) == live_paths(py_uri)+ assert content(lite_uri) == content(py_uri) == expected+ commit = deltalake.DeltaTable(lite_uri).history(1)[0]+ assert commit["operation"] == "RESTORE"+ assert commit["rolled_back_run_uuid"] == "run-1"+ # The handle moved onto the restore commit, so later writes through it see the restored table.+ assert lite.version() == lite_metrics["version"]+ assert sorted(f["path"] for f in lite.files()) == live_paths(lite_uri)+++ "log_files",+ [+ pytest.param({}, id="empty_log"),+ pytest.param({"00000000000000000000.json": "not a commit"}, id="damaged_commit"),+ ],+)+def test_table_exists_agrees_with_delta_rs_on_a_log_without_a_loadable_table(tmp_path, log_files):+ import deltalite++ log = tmp_path / "t" / "_delta_log"+ log.mkdir(parents=True)+ for name, body in log_files.items():+ (log / name).write_text(body)+ uri = str(tmp_path / "t")++ assert deltalite.DeltaLiteTable.table_exists(uri) is deltalake.DeltaTable.is_deltatable(uri)+++def test_table_exists_raises_on_a_storage_error():+ import deltalite++ unreachable = {+ "AWS_ENDPOINT_URL": "http://127.0.0.1:1",+ "AWS_ALLOW_HTTP": "true",+ "AWS_ACCESS_KEY_ID": "test",+ "AWS_SECRET_ACCESS_KEY": "test",+ "AWS_REGION": "us-east-1",+ "max_retries": "0",+ }++ with pytest.raises(deltalite.DeltaLiteError):+ deltalite.DeltaLiteTable.table_exists("s3://bucket/table", storage_options=unreachable)