25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。
TypeScript aicliclickhousedatabasedatabase-clientdatabase-managementdockerguimcpmongodbmysqlpostgresqlredisrustsql-serversqlitetaurivue
use std::collections::HashSet;
use std::sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc,
};
use std::time::{Duration, Instant};
use mongodb::bson::{Document, RawDocumentBuf};
use tokio::sync::mpsc;
use super::{
archive, cancelled, source::CatalogSource, CancelFuture, DumpClient, MongoDatabaseDumpProgress,
MongoDatabaseRestoreRequest, MongoDumpFormat, PreparedCollection,
};
use crate::connection::AppState;
const BATCH_BYTES: usize = 8 * 1024 * 1024;
enum Event {
Batch { index: usize, documents: Vec<RawDocumentBuf> },
End(usize),
}
struct Batcher {
tx: mpsc::Sender<Event>,
index: usize,
documents: Vec<RawDocumentBuf>,
bytes: usize,
limit: usize,
objcheck: bool,
}
impl Batcher {
fn flush(&mut self) -> Result<(), String> {
if self.documents.is_empty() {
return Ok(());
}
let documents = std::mem::take(&mut self.documents);
self.bytes = 0;
self.tx
.blocking_send(Event::Batch { index: self.index, documents })
.map_err(|_| "MongoDB dump/restore cancelled".into())
}
fn push(&mut self, index: usize, bytes: Vec<u8>) -> Result<(), String> {
if index != self.index || self.bytes.saturating_add(bytes.len()) > BATCH_BYTES {
self.flush()?;
}
self.index = index;
let document = checked_document(bytes, self.objcheck)?;
self.bytes += document.as_bytes().len();
self.documents.push(document);
if self.documents.len() >= self.limit {
self.flush()?;
}
Ok(())
}
fn end(&mut self, index: usize) -> Result<(), String> {
self.flush()?;
self.tx.blocking_send(Event::End(index)).map_err(|_| "MongoDB dump/restore cancelled".into())
}
}
// Like mongorestore's objcheck, deep parsing is optional and occurs in the stream,
// immediately before a document is queued for insertion. No whole-backup pass.
fn checked_document(bytes: Vec<u8>, objcheck: bool) -> Result<RawDocumentBuf, String> {
let raw = RawDocumentBuf::from_bytes(bytes).map_err(|e| format!("Invalid BSON document: {e}"))?;
if objcheck {
mongodb::bson::from_slice::<Document>(raw.as_bytes()).map_err(|e| format!("Invalid BSON document: {e}"))?;
}
Ok(raw)
}
fn produce(
source: Arc<CatalogSource>,
entries: Vec<PreparedCollection>,
flag: Arc<AtomicBool>,
bytes_read: Arc<AtomicU64>,
mut batcher: Batcher,
) -> Result<(), String> {
let result = (|| {
source.check_ready(&entries)?;
if source.format() == MongoDumpFormat::Archive {
let indices = source
.entries
.iter()
.map(|original| {
entries.iter().position(|e| {
e.database == original.database
&& e.metadata.collection_name == original.metadata.collection_name
&& !e.metadata.is_view()
})
})
.collect::<Vec<_>>();
archive::stream(
&mut *source.reader(None)?,
&source.entries,
|| cancelled(&flag),
|event| {
match event {
archive::StreamEvent::Document(i, bytes) => {
bytes_read.fetch_add(bytes.len() as u64, Ordering::Relaxed);
if let Some(index) = indices[i] {
batcher.push(index, bytes)?;
}
}
archive::StreamEvent::End(i) => {
if let Some(index) = indices[i] {
batcher.end(index)?;
}
}
}
Ok(())
},
)?;
} else {
for (index, entry) in entries.iter().enumerate().filter(|(_, e)| !e.metadata.is_view()) {
cancelled(&flag)?;
let path = entry.path.as_deref().ok_or("Missing collection BSON file")?;
let mut reader = source.reader(Some(path))?;
loop {
cancelled(&flag)?;
match archive::read_raw_frame(&mut *reader)? {
archive::Frame::End => break,
archive::Frame::Terminator => return Err("Unexpected terminator in collection BSON".into()),
archive::Frame::Document(bytes) => {
bytes_read.fetch_add(bytes.len() as u64, Ordering::Relaxed);
batcher.push(index, bytes)?;
}
}
}
batcher.end(index)?;
}
}
Ok(())
})();
// Deliver valid earlier documents even if a later frame is corrupt, matching
// the streaming import contract and reporting partial writes accurately.
let flushed = batcher.flush();
result.and(flushed)
}
pub(super) async fn initialize_collection(
client: &DumpClient,
request: &MongoDatabaseRestoreRequest,
entry: &PreparedCollection,
exists: bool,
) -> Result<(), String> {
if request.drop_existing && exists {
client.drop_collection(&request.database, &entry.metadata.collection_name).await?;
}
if !exists || request.drop_existing {
client.create_collection(&request.database, entry.metadata.create_command(request.restore_options)).await?;
}
Ok(())
}
pub(super) async fn writable(state: &AppState, id: &str) -> Result<(), String> {
if let Some(name) = crate::query::connection_readonly_name(state, id).await {
return Err(format!("Read-only connection '{name}': restore blocked"));
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(super) async fn restore_data(
state: &AppState,
client: &DumpClient,
request: &MongoDatabaseRestoreRequest,
source: Arc<CatalogSource>,
entries: &[PreparedCollection],
existing: &HashSet<String>,
progress: &mut MongoDatabaseDumpProgress,
started: Instant,
is_cancelled: &mut impl FnMut(&str) -> CancelFuture,
on_progress: &mut impl FnMut(MongoDatabaseDumpProgress),
) -> Result<(), String> {
struct StopOnDrop(Arc<AtomicBool>);
impl Drop for StopOnDrop {
fn drop(&mut self) {
self.0.store(true, Ordering::Relaxed);
}
}
let flag = Arc::new(AtomicBool::new(false));
let _guard = StopOnDrop(flag.clone());
let (tx, mut rx) = mpsc::channel(2);
let batcher = Batcher {
tx,
index: 0,
documents: Vec::new(),
bytes: 0,
limit: request.batch_size.max(1),
objcheck: request.objcheck,
};
let producer_entries = entries.to_vec();
let signal = flag.clone();
let bytes_read = Arc::new(AtomicU64::new(0));
let reader_progress = bytes_read.clone();
let producer =
tokio::task::spawn_blocking(move || produce(source, producer_entries, signal, reader_progress, batcher));
progress.phase = "data".into();
progress.emit(started, on_progress);
let result = async {
let mut initialized = HashSet::new();
let mut tick = tokio::time::interval(Duration::from_millis(100));
loop {
let event = tokio::select! {
event = rx.recv() => match event { Some(event) => event, None => break },
_ = tick.tick() => {
if is_cancelled(&request.task_id).await { return Err("MongoDB dump/restore cancelled".into()); }
progress.bytes_processed = bytes_read.load(Ordering::Relaxed);
progress.emit(started, on_progress);
continue;
},
};
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
writable(state, &request.connection_id).await?;
let index = match &event {
Event::Batch { index, .. } | Event::End(index) => *index,
};
let entry = &entries[index];
progress.phase = "data".into();
progress.collection = Some(entry.metadata.collection_name.clone());
progress.emit(started, on_progress);
if initialized.insert(index) {
initialize_collection(client, request, entry, existing.contains(&entry.metadata.collection_name))
.await?;
}
match event {
Event::Batch { documents, .. } => {
progress.documents_read += documents.len() as u64;
progress.bytes_processed = bytes_read.load(Ordering::Relaxed);
if request.objcheck {
progress.documents_validated += documents.len() as u64;
}
let outcome =
client.insert_documents(&request.database, &entry.metadata.collection_name, documents).await?;
progress.documents_written += outcome.inserted;
progress.documents_failed += outcome.errors.len() as u64;
progress.emit(started, on_progress);
if let Some(error) = outcome.errors.first().filter(|_| request.stop_on_error) {
return Err(error.message.clone());
}
}
Event::End(_) => {
progress.collections_done += 1;
progress.emit(started, on_progress);
}
}
}
Ok(())
}
.await;
flag.store(true, Ordering::Relaxed);
drop(rx);
let produced = producer.await.map_err(|e| e.to_string()).and_then(|r| r);
result.and(produced)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn objcheck_uses_official_bson_parser_only_when_enabled() {
// Valid outer framing but invalid boolean payload.
let bytes = vec![9, 0, 0, 0, 8, b'x', 0, 2, 0];
assert!(checked_document(bytes.clone(), false).is_ok());
assert!(checked_document(bytes, true).is_err());
assert!(checked_document(vec![5, 0, 0, 0, 1], false).is_err());
}
#[test]
fn raw_bson_preserves_duplicate_keys_and_exact_bytes() {
let bytes = vec![19, 0, 0, 0, 16, b'x', 0, 1, 0, 0, 0, 16, b'x', 0, 2, 0, 0, 0, 0];
for objcheck in [false, true] {
let raw = checked_document(bytes.clone(), objcheck).unwrap();
assert_eq!(raw.as_bytes(), bytes);
assert_eq!(mongodb::bson::to_vec(&raw).unwrap(), bytes);
}
}
#[tokio::test]
async fn reader_delivers_earlier_batches_before_reporting_corruption() {
use crate::mongodb_dump::{
prepare_mongodb_restore_source, release_mongodb_restore_source, MongoRestoreSourceRequest,
};
let dir = tempfile::tempdir().unwrap();
let db = dir.path().join("source");
std::fs::create_dir(&db).unwrap();
let mut bytes = mongodb::bson::to_vec(&mongodb::bson::doc! { "_id": 1 }).unwrap();
bytes.extend_from_slice(b"invalid BSON");
std::fs::write(db.join("records.bson"), bytes).unwrap();
let preview = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
path: db.to_string_lossy().into_owned(),
format: MongoDumpFormat::Directory,
gzip: false,
})
.await
.unwrap();
let source = super::super::source::get(&preview.source_ref).unwrap();
let entries = source.entries.clone();
let (tx, mut rx) = mpsc::channel(1);
let batcher = Batcher { tx, index: 0, documents: Vec::new(), bytes: 0, limit: 1, objcheck: false };
let worker = tokio::task::spawn_blocking(move || {
produce(source, entries, Arc::new(AtomicBool::new(false)), Arc::new(AtomicU64::new(0)), batcher)
});
assert!(matches!(rx.recv().await, Some(Event::Batch { documents, .. }) if documents.len() == 1));
assert!(worker.await.unwrap().unwrap_err().contains("BSON"));
assert!(rx.recv().await.is_none());
release_mongodb_restore_source(&preview.source_ref);
}
#[test]
fn batches_are_limited_by_bytes_as_well_as_document_count() {
let bytes = mongodb::bson::to_vec(&mongodb::bson::doc! { "payload": "x".repeat(5 * 1024 * 1024) }).unwrap();
let (tx, mut rx) = mpsc::channel(2);
let mut batcher = Batcher { tx, index: 0, documents: Vec::new(), bytes: 0, limit: 500, objcheck: false };
batcher.push(0, bytes.clone()).unwrap();
batcher.push(0, bytes).unwrap();
batcher.flush().unwrap();
for _ in 0..2 {
assert!(matches!(rx.try_recv().unwrap(), Event::Batch { documents, .. } if documents.len() == 1));
}
}
#[test]
#[ignore = "opt-in compressed BSON parser throughput smoke test"]
fn compressed_bson_parser_throughput() {
use flate2::{read::MultiGzDecoder, write::GzEncoder, Compression};
use std::io::{BufReader, Cursor, Write};
let mut document = Document::new();
document.insert("_id", 1i64);
for i in 0..64 {
document.insert(format!("field_{i}"), "value".repeat(8));
}
let encoded = mongodb::bson::to_vec(&document).unwrap();
let documents = 8192;
let mut encoder = GzEncoder::new(Vec::new(), Compression::fast());
for _ in 0..documents {
encoder.write_all(&encoded).unwrap();
}
let compressed = encoder.finish().unwrap();
for (mode, objcheck, passes) in
[("raw-stream", false, 1), ("objcheck-stream", true, 1), ("old-double-parse", true, 2)]
{
let started = Instant::now();
let mut count = 0;
for _ in 0..passes {
let mut input = BufReader::with_capacity(64 * 1024, MultiGzDecoder::new(Cursor::new(&compressed)));
loop {
match archive::read_raw_frame(&mut input).unwrap() {
archive::Frame::Document(bytes) => {
checked_document(bytes, objcheck).unwrap();
count += 1;
}
archive::Frame::End => break,
_ => panic!("unexpected terminator"),
}
}
}
assert_eq!(count, documents * passes);
eprintln!(
"{mode}: logical_mib={:.2}, elapsed_ms={}, logical_mib_s={:.2}",
(documents * encoded.len()) as f64 / 1048576.0,
started.elapsed().as_millis(),
(documents * encoded.len()) as f64 / 1048576.0 / started.elapsed().as_secs_f64()
);
}
}
}