Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(deltalite): add a GIL-free restore and a delta-rs table check (#119677)

Daniel RCafdd0d1f267a
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 92d790a200c8..e96ef7757838 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -3982,7 +3982,7 @@ dependencies = [
[[package]]
name = "deltalite-python"
-version = "0.1.12"
+version = "0.1.13"
dependencies = [
"arrow",
"arrow-array",
diff --git a/rust/deltalite/core/src/handle.rs b/rust/deltalite/core/src/handle.rs
index 42745698c692..ded4753160bd 100644
--- a/rust/deltalite/core/src/handle.rs
+++ b/rust/deltalite/core/src/handle.rs
@@ -23,9 +23,12 @@ use std::time::{Duration, Instant};
use arrow_array::RecordBatch;
use arrow_schema::SchemaRef;
+use deltalake::kernel::transaction::CommitProperties;
+use deltalake::operations::restore::RestoreMetrics;
use deltalake::table::config::TablePropertiesExt as _;
use deltalake::DeltaTable;
use metrics::counter;
+use serde_json::Value;
use crate::errors::{Error, Result};
use crate::limits::{env_switch, switch_setting};
@@ -434,6 +437,42 @@ impl TableHandle {
Ok(stats)
}
+ /// Fails when a vacuum deleted a file of `version`.
+ pub async fn restore(
+ &mut self,
+ version: u64,
+ commit_metadata: Option<HashMap<String, Value>>,
+ ) -> Result<RestoreMetrics> {
+ let result = self.restore_refreshed(version, commit_metadata).await;
+ self.prefetch.clear();
+ result
+ }
+
+ async fn restore_refreshed(
+ &mut self,
+ version: u64,
+ commit_metadata: Option<HashMap<String, Value>>,
+ ) -> Result<RestoreMetrics> {
+ self.refresh().await?;
+ let mut builder = self
+ .table
+ .clone()
+ .restore()
+ .with_version_to_restore(version);
+ if let Some(metadata) = commit_metadata {
+ builder =
+ builder.with_commit_properties(CommitProperties::default().with_metadata(metadata));
+ }
+ let (table, metrics) = builder.await?;
+ if self.adopt_commit_snapshot {
+ self.table.state = table.state;
+ self.list_on_next_refresh = self.on_checkpoint_boundary();
+ } else {
+ self.refresh().await?;
+ }
+ Ok(metrics)
+ }
+
/// The open cost, handed out once so per-upsert stats sum to the handle's total.
fn take_initial_open_ms(&mut self) -> u64 {
if self.initial_open_reported {
diff --git a/rust/deltalite/core/src/table.rs b/rust/deltalite/core/src/table.rs
index c4bb28f280a9..1cc4258f79e5 100644
--- a/rust/deltalite/core/src/table.rs
+++ b/rust/deltalite/core/src/table.rs
@@ -59,6 +59,19 @@ pub async fn open_table(uri: &str, storage_options: HashMap<String, String>) ->
Ok(table)
}
+/// `deltalake.DeltaTable.is_deltatable`'s check. It loads nothing, so a table with a damaged
+/// log still counts as present.
+pub async fn table_exists(uri: &str, storage_options: HashMap<String, String>) -> Result<bool> {
+ let Ok(url) = deltalake::ensure_table_uri(uri) else {
+ return Ok(false);
+ };
+ let Ok(builder) = DeltaTableBuilder::from_url(url) else {
+ return Ok(false);
+ };
+ let table = builder.with_storage_options(storage_options).build()?;
+ Ok(table.verify_deltatable_existence().await?)
+}
+
/// [`open_table`] that also hands back the checkpoint cache the table's stores serve
/// from, still holding the checkpoint bytes read by this load. The caller clears it once
/// it is done and can clear it again after each later load through the same table.
diff --git a/rust/deltalite/python/Cargo.toml b/rust/deltalite/python/Cargo.toml
index 2dc94b04c601..54ba588631d4 100644
--- a/rust/deltalite/python/Cargo.toml
+++ b/rust/deltalite/python/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "deltalite-python"
# Mirrors pyproject.toml in this directory (always identical); bump both together.
-version = "0.1.12"
+version = "0.1.13"
edition = "2021"
# CI compiles with this exact toolchain; cargo's MSRV-aware resolver uses it to
# avoid picking dependency versions CI cannot build (aws-* crates move MSRV often).
diff --git a/rust/deltalite/python/README.md b/rust/deltalite/python/README.md
index 08e5d1e9ea46..034b3da9fcbb 100644
--- a/rust/deltalite/python/README.md
+++ b/rust/deltalite/python/README.md
@@ -100,8 +100,10 @@ rejected (raising `DeltaLiteError`) rather than silently double-inserted.
|---|---|
| `DeltaLiteTable.open(uri, storage_options=None)` | Open an existing Delta table. `storage_options` is the usual object-store dict (S3/GCS/Azure/local). |
| `DeltaLiteTable.is_deltatable(uri, storage_options=None)` | `True` if a Delta table exists at `uri`. |
+| `DeltaLiteTable.table_exists(uri, storage_options=None)` | `True` if the log at `uri` holds a commit or a checkpoint, the same check as `DeltaTable.is_deltatable`. Storage errors raise; they never read as `False`. |
| `.upsert(data, primary_keys, partition_key=None, **opts)` | Insert-or-replace `data` by key. Returns `UpsertStats`. See knobs below. |
| `.compact(**opts)` | Small-file compaction, the replacement for `DeltaTable.optimize.compact`. Returns a stats dict. See "Compaction" below. |
+| `.restore(version, *, commit_metadata=None)` | delta-rs's restore to `version`, in one commit, with the GIL released. Returns `numRemovedFile`, `numRestoredFile` and the committed `version`. The handle moves onto the restored table. |
| `.version()` | Current table version (`int`). |
| `.reload()` | Bring the table state up to date with the log (incremental; falls back to a full re-open). |
| `.table_id()` | The table id from the metadata action (`str`). |
diff --git a/rust/deltalite/python/pyproject.toml b/rust/deltalite/python/pyproject.toml
index d8ce1097b214..f1102f1387af 100644
--- a/rust/deltalite/python/pyproject.toml
+++ b/rust/deltalite/python/pyproject.toml
@@ -10,7 +10,7 @@ build-backend = "maturin"
[project]
name = "deltalite"
# Mirrors `rust/deltalite/python/Cargo.toml` (always identical); bump both together.
-version = "0.1.12"
+version = "0.1.13"
description = "Streaming partition upsert for Delta tables, replacing delta-rs SQL MERGE"
readme = "README.md"
requires-python = ">=3.12"
diff --git a/rust/deltalite/python/src/lib.rs b/rust/deltalite/python/src/lib.rs
index d5bbdea8bac5..b7a440eedd6d 100644
--- a/rust/deltalite/python/src/lib.rs
+++ b/rust/deltalite/python/src/lib.rs
@@ -16,7 +16,7 @@ use deltalite_core::errors::Error;
use deltalite_core::handle::TableHandle;
use deltalite_core::limits::ProcessLimits;
use deltalite_core::schema::import_column;
-use deltalite_core::table::{open_table, MultipartConfig};
+use deltalite_core::table::{open_table, table_exists, MultipartConfig};
use deltalite_core::upsert::{PruneStrategy, UpsertOptions};
use pyo3::create_exception;
use pyo3::exceptions::PyException;
@@ -268,6 +268,20 @@ impl DeltaLiteTable {
.is_ok())
}
+ /// `deltalake.DeltaTable.is_deltatable`'s check. Unlike `is_deltatable`, storage errors
+ /// raise: a caller that reads "no table" can create a new table over an existing one.
+ #[staticmethod]
+ #[pyo3(signature = (uri, storage_options = None))]
+ fn table_exists(
+ py: Python<'_>,
+ uri: String,
+ storage_options: Option<HashMap<String, String>>,
+ ) -> PyResult<bool> {
+ let so = storage_options.unwrap_or_default();
+ py.detach(|| runtime().block_on(table_exists(&uri, so)))
+ .map_err(to_py_err)
+ }
+
/// The table version this handle currently observes (-1 before any load).
fn version(&self) -> i64 {
self.handle.version()
@@ -594,6 +608,30 @@ impl DeltaLiteTable {
Ok(d.into_any().unbind())
}
+ /// Returns the keys delta-rs's `DeltaTable.restore` returns, plus the committed `version`.
+ #[pyo3(signature = (version, *, commit_metadata = None))]
+ fn restore(
+ &mut self,
+ py: Python<'_>,
+ version: u64,
+ commit_metadata: Option<HashMap<String, String>>,
+ ) -> PyResult<Py<PyAny>> {
+ let metadata = commit_metadata.map(|m| {
+ m.into_iter()
+ .map(|(k, v)| (k, Value::String(v)))
+ .collect::<HashMap<String, Value>>()
+ });
+ let handle = &mut self.handle;
+ let metrics = py
+ .detach(|| runtime().block_on(handle.restore(version, metadata)))
+ .map_err(to_py_err)?;
+ let d = PyDict::new(py);
+ d.set_item("numRemovedFile", metrics.num_removed_file)?;
+ d.set_item("numRestoredFile", metrics.num_restored_file)?;
+ d.set_item("version", self.handle.version())?;
+ Ok(d.into_any().unbind())
+ }
+
/// Commit metadata of the most recent `limit` commits, oldest first.
fn history(&self, py: Python<'_>, limit: usize) -> PyResult<Py<PyAny>> {
let table = self.handle.table().clone();
diff --git a/rust/deltalite/python/tests/test_operations.py b/rust/deltalite/python/tests/test_operations.py
index 885edeed70a4..f9fd85619b4b 100644
--- a/rust/deltalite/python/tests/test_operations.py
+++ b/rust/deltalite/python/tests/test_operations.py
@@ -11,6 +11,7 @@ from __future__ import annotations
import sys
import json
+import shutil
from pathlib import Path
import pytest
@@ -35,7 +36,7 @@ def simple(ids, part, v, extra="ABSENT"):
return pa.table(d)
-def content(uri) -> dict[tuple[str | None, str], int]:
+def content(uri) -> dict[tuple[str | None, str | None], int | None]:
"""{(id, partition): v} via the Python package -- also proves the log is readable."""
tbl = deltalake.DeltaTable(uri).to_pyarrow_table()
return dict(
@@ -971,3 +972,79 @@ def test_upsert_creates_checkpoint_at_interval_boundary(tmp_path):
# An off-boundary commit must not add another checkpoint.
upsert_path(uri, simple(["c"], "p", 3), ["id"], PARTITION_KEY)
assert list(log.glob("*.checkpoint.parquet")) == checkpoints
+
+
+def live_paths(uri) -> list[str]:
+ root = Path(uri).resolve()
+ return sorted(str(Path(u).relative_to(root)) for u in deltalake.DeltaTable(uri).file_uris())
+
+
+def test_restore_matches_delta_rs_restore(tmp_path):
+ import deltalite
+
+ p = "2026-07-23"
+ base = tmp_path / "base"
+ create_table(str(base), simple(["a", "b"], p, 1), True)
+ upsert_path(str(base), simple(["c"], p, 2), ["id"], PARTITION_KEY)
+ target = deltalake.DeltaTable(str(base)).version()
+ expected = content(str(base))
+ upsert_path(str(base), simple(["a", "d"], p, 3), ["id"], PARTITION_KEY)
+ deltalake.DeltaTable(str(base)).optimize.compact()
+ py_uri, lite_uri = str(tmp_path / "py"), str(tmp_path / "lite")
+ shutil.copytree(base, py_uri)
+ shutil.copytree(base, lite_uri)
+ metadata = {"rolled_back_run_uuid": "run-1"}
+
+ py_metrics = deltalake.DeltaTable(py_uri).restore(
+ target, commit_properties=deltalake.CommitProperties(custom_metadata=metadata)
+ )
+ lite = deltalite.DeltaLiteTable.open(lite_uri)
+ lite_metrics = lite.restore(target, commit_metadata=metadata)
+
+ assert py_metrics["numRestoredFile"] > 0
+ assert lite_metrics["numRemovedFile"] == py_metrics["numRemovedFile"]
+ assert lite_metrics["numRestoredFile"] == py_metrics["numRestoredFile"]
+ assert lite_metrics["version"] == deltalake.DeltaTable(py_uri).version()
+ assert live_paths(lite_uri) == live_paths(py_uri)
+ assert content(lite_uri) == content(py_uri) == expected
+ commit = deltalake.DeltaTable(lite_uri).history(1)[0]
+ assert commit["operation"] == "RESTORE"
+ assert commit["rolled_back_run_uuid"] == "run-1"
+ # The handle moved onto the restore commit, so later writes through it see the restored table.
+ assert lite.version() == lite_metrics["version"]
+ assert sorted(f["path"] for f in lite.files()) == live_paths(lite_uri)
+
+
+ "log_files",
+ [
+ pytest.param({}, id="empty_log"),
+ pytest.param({"00000000000000000000.json": "not a commit"}, id="damaged_commit"),
+ ],
+)
+def test_table_exists_agrees_with_delta_rs_on_a_log_without_a_loadable_table(tmp_path, log_files):
+ import deltalite
+
+ log = tmp_path / "t" / "_delta_log"
+ log.mkdir(parents=True)
+ for name, body in log_files.items():
+ (log / name).write_text(body)
+ uri = str(tmp_path / "t")
+
+ assert deltalite.DeltaLiteTable.table_exists(uri) is deltalake.DeltaTable.is_deltatable(uri)
+
+
+def test_table_exists_raises_on_a_storage_error():
+ import deltalite
+
+ unreachable = {
+ "AWS_ENDPOINT_URL": "http://127.0.0.1:1",
+ "AWS_ALLOW_HTTP": "true",
+ "AWS_ACCESS_KEY_ID": "test",
+ "AWS_SECRET_ACCESS_KEY": "test",
+ "AWS_REGION": "us-east-1",
+ "max_retries": "0",
+ }
+
+ with pytest.raises(deltalite.DeltaLiteError):
+ deltalite.DeltaLiteTable.table_exists("s3://bucket/table", storage_options=unreachable)