Skip to content

Bring projects, wikis, and teams together with AI. AppFlowy is the AI collaborative workspace where you achieve more without losing control of your data. The leading open source Notion alternative.

fix: stuck when importing large csv file (#4878)

Nathan.fooo9c3be7e4885a
diff --git a/frontend/appflowy_tauri/src-tauri/Cargo.lock b/frontend/appflowy_tauri/src-tauri/Cargo.lock
index 221283296f..4b556d9825 100644
--- a/frontend/appflowy_tauri/src-tauri/Cargo.lock
+++ b/frontend/appflowy_tauri/src-tauri/Cargo.lock
@@ -818,7 +818,7 @@ dependencies = [
[[package]]
name = "collab"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"async-trait",
@@ -841,7 +841,7 @@ dependencies = [
[[package]]
name = "collab-database"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"async-trait",
@@ -855,6 +855,7 @@ dependencies = [
"lru",
"nanoid",
"parking_lot 0.12.1",
+ "rayon",
"serde",
"serde_json",
"serde_repr",
@@ -870,7 +871,7 @@ dependencies = [
[[package]]
name = "collab-document"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"collab",
@@ -889,7 +890,7 @@ dependencies = [
[[package]]
name = "collab-entity"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"bytes",
@@ -904,7 +905,7 @@ dependencies = [
[[package]]
name = "collab-folder"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"chrono",
@@ -941,7 +942,7 @@ dependencies = [
[[package]]
name = "collab-plugins"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"async-stream",
@@ -980,7 +981,7 @@ dependencies = [
[[package]]
name = "collab-user"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"collab",
@@ -1203,7 +1204,7 @@ dependencies = [
"cssparser-macros",
"dtoa-short",
"itoa 1.0.6",
- "phf 0.8.0",
+ "phf 0.11.2",
"smallvec",
]
@@ -4781,9 +4782,9 @@ checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9"
[[package]]
name = "rayon"
-version = "1.8.1"
+version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051"
+checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd"
dependencies = [
"either",
"rayon-core",
diff --git a/frontend/appflowy_tauri/src-tauri/Cargo.toml b/frontend/appflowy_tauri/src-tauri/Cargo.toml
index 5afe966665..6ba93681f6 100644
--- a/frontend/appflowy_tauri/src-tauri/Cargo.toml
+++ b/frontend/appflowy_tauri/src-tauri/Cargo.toml
@@ -92,10 +92,10 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "0be
# To switch to the local path, run:
# scripts/tool/update_collab_source.sh
# ⚠️⚠️⚠️️
-collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
+collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
diff --git a/frontend/appflowy_web/wasm-libs/Cargo.toml b/frontend/appflowy_web/wasm-libs/Cargo.toml
index abd5e3752b..7be865bfab 100644
--- a/frontend/appflowy_web/wasm-libs/Cargo.toml
+++ b/frontend/appflowy_web/wasm-libs/Cargo.toml
@@ -65,10 +65,10 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "0be
# To switch to the local path, run:
# scripts/tool/update_collab_source.sh
# ⚠️⚠️⚠️️
-collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
+collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock
index b43888daa0..992af39b56 100644
--- a/frontend/rust-lib/Cargo.lock
+++ b/frontend/rust-lib/Cargo.lock
@@ -746,7 +746,7 @@ dependencies = [
[[package]]
name = "collab"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"async-trait",
@@ -769,7 +769,7 @@ dependencies = [
[[package]]
name = "collab-database"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"async-trait",
@@ -783,6 +783,7 @@ dependencies = [
"lru",
"nanoid",
"parking_lot 0.12.1",
+ "rayon",
"serde",
"serde_json",
"serde_repr",
@@ -798,7 +799,7 @@ dependencies = [
[[package]]
name = "collab-document"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"collab",
@@ -817,7 +818,7 @@ dependencies = [
[[package]]
name = "collab-entity"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"bytes",
@@ -832,7 +833,7 @@ dependencies = [
[[package]]
name = "collab-folder"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"chrono",
@@ -869,7 +870,7 @@ dependencies = [
[[package]]
name = "collab-plugins"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"async-stream",
@@ -908,7 +909,7 @@ dependencies = [
[[package]]
name = "collab-user"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=9b189e7dc180ddc7fd79c49ed16f9c8a46216380#9b189e7dc180ddc7fd79c49ed16f9c8a46216380"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=3079e1cef184bf5ddfecbc6999a450ae9ac070a7#3079e1cef184bf5ddfecbc6999a450ae9ac070a7"
dependencies = [
"anyhow",
"collab",
@@ -4290,9 +4291,9 @@ dependencies = [
[[package]]
name = "rayon"
-version = "1.8.1"
+version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051"
+checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd"
dependencies = [
"either",
"rayon-core",
diff --git a/frontend/rust-lib/Cargo.toml b/frontend/rust-lib/Cargo.toml
index d098d11338..82b32f10d0 100644
--- a/frontend/rust-lib/Cargo.toml
+++ b/frontend/rust-lib/Cargo.toml
@@ -115,10 +115,10 @@ client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "0be
# To switch to the local path, run:
# scripts/tool/update_collab_source.sh
# ⚠️⚠️⚠️️
-collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
-collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "9b189e7dc180ddc7fd79c49ed16f9c8a46216380" }
+collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-database = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-plugins = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-user = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
+collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "3079e1cef184bf5ddfecbc6999a450ae9ac070a7" }
diff --git a/frontend/rust-lib/event-integration/src/folder_event.rs b/frontend/rust-lib/event-integration/src/folder_event.rs
index af426d1615..4f3d218e26 100644
--- a/frontend/rust-lib/event-integration/src/folder_event.rs
+++ b/frontend/rust-lib/event-integration/src/folder_event.rs
@@ -8,6 +8,7 @@ use flowy_user::entities::{
};
use flowy_user::errors::FlowyError;
use flowy_user::event_map::UserEvent;
+use tokio::time::sleep;
use crate::event_builder::EventBuilder;
use crate::EventIntegrationTest;
@@ -134,6 +135,15 @@ impl EventIntegrationTest {
.await
.parse::<ViewPB>()
}
+
+ pub async fn import_data(&self, data: ImportPB) -> ViewPB {
+ EventBuilder::new(self.clone())
+ .event(FolderEvent::ImportData)
+ .payload(data)
+ .async_send()
+ .await
+ .parse::<ViewPB>()
+ }
}
pub struct ViewTest {
@@ -172,15 +182,23 @@ impl ViewTest {
}
pub async fn new_grid_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self {
- Self::new(sdk, ViewLayoutPB::Grid, data).await
+ // TODO(nathan): remove this sleep
+ // workaround for the rows that are created asynchronously
+ let this = Self::new(sdk, ViewLayoutPB::Grid, data).await;
+ sleep(tokio::time::Duration::from_secs(2)).await;
+ this
}
pub async fn new_board_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self {
- Self::new(sdk, ViewLayoutPB::Board, data).await
+ let this = Self::new(sdk, ViewLayoutPB::Board, data).await;
+ sleep(tokio::time::Duration::from_secs(2)).await;
+ this
}
pub async fn new_calendar_view(sdk: &EventIntegrationTest, data: Vec<u8>) -> Self {
- Self::new(sdk, ViewLayoutPB::Calendar, data).await
+ let this = Self::new(sdk, ViewLayoutPB::Calendar, data).await;
+ sleep(tokio::time::Duration::from_secs(2)).await;
+ this
}
}
diff --git a/frontend/rust-lib/event-integration/tests/asset/csv_10240r_15c.csv.zip b/frontend/rust-lib/event-integration/tests/asset/csv_10240r_15c.csv.zip
new file mode 100644
index 0000000000..9d6b57efc3
Binary files /dev/null and b/frontend/rust-lib/event-integration/tests/asset/csv_10240r_15c.csv.zip differ
diff --git a/frontend/rust-lib/event-integration/tests/asset/csv_492r_17c.csv.zip b/frontend/rust-lib/event-integration/tests/asset/csv_492r_17c.csv.zip
new file mode 100644
index 0000000000..0d38422ac3
Binary files /dev/null and b/frontend/rust-lib/event-integration/tests/asset/csv_492r_17c.csv.zip differ
diff --git a/frontend/rust-lib/event-integration/tests/folder/local_test/import_test.rs b/frontend/rust-lib/event-integration/tests/folder/local_test/import_test.rs
new file mode 100644
index 0000000000..7b812fc821
--- /dev/null
+++ b/frontend/rust-lib/event-integration/tests/folder/local_test/import_test.rs
@@ -0,0 +1,55 @@
+use crate::util::unzip_history_user_db;
+use event_integration::EventIntegrationTest;
+use flowy_core::DEFAULT_NAME;
+use flowy_folder::entities::{ImportPB, ImportTypePB, ViewLayoutPB};
+
+#[tokio::test]
+async fn import_492_row_csv_file_test() {
+ // csv_500r_15c.csv is a file with 492 rows and 17 columns
+ let file_name = "csv_492r_17c.csv".to_string();
+ let (cleaner, csv_file_path) = unzip_history_user_db("./tests/asset", &file_name).unwrap();
+
+ let csv_string = std::fs::read_to_string(csv_file_path).unwrap();
+ let test = EventIntegrationTest::new_with_name(DEFAULT_NAME).await;
+ test.sign_up_as_guest().await;
+
+ let workspace_id = test.get_current_workspace().await.id;
+ let import_data = gen_import_data(file_name, csv_string, workspace_id);
+
+ let view = test.import_data(import_data).await;
+ let database = test.get_database(&view.id).await;
+ assert_eq!(database.rows.len(), 492);
+ drop(cleaner);
+}
+
+#[tokio::test]
+async fn import_10240_row_csv_file_test() {
+ // csv_22577r_15c.csv is a file with 10240 rows and 15 columns
+ let file_name = "csv_10240r_15c.csv".to_string();
+ let (cleaner, csv_file_path) = unzip_history_user_db("./tests/asset", &file_name).unwrap();
+
+ let csv_string = std::fs::read_to_string(csv_file_path).unwrap();
+ let test = EventIntegrationTest::new_with_name(DEFAULT_NAME).await;
+ test.sign_up_as_guest().await;
+
+ let workspace_id = test.get_current_workspace().await.id;
+ let import_data = gen_import_data(file_name, csv_string, workspace_id);
+
+ let view = test.import_data(import_data).await;
+ let database = test.get_database(&view.id).await;
+ assert_eq!(database.rows.len(), 10240);
+
+ drop(cleaner);
+}
+
+fn gen_import_data(file_name: String, csv_string: String, workspace_id: String) -> ImportPB {
+ let import_data = ImportPB {
+ parent_view_id: workspace_id.clone(),
+ name: file_name,
+ data: Some(csv_string.as_bytes().to_vec()),
+ file_path: None,
+ view_layout: ViewLayoutPB::Grid,
+ import_type: ImportTypePB::CSV,
+ };
+ import_data
+}
diff --git a/frontend/rust-lib/event-integration/tests/folder/local_test/mod.rs b/frontend/rust-lib/event-integration/tests/folder/local_test/mod.rs
index e13c3c1e76..aa58a02baf 100644
--- a/frontend/rust-lib/event-integration/tests/folder/local_test/mod.rs
+++ b/frontend/rust-lib/event-integration/tests/folder/local_test/mod.rs
@@ -1,4 +1,5 @@
mod folder_test;
+mod import_test;
mod script;
mod subscription_test;
mod test;
diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs
index 4becb36241..d422478923 100644
--- a/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs
+++ b/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps.rs
@@ -360,8 +360,11 @@ impl FolderOperationHandler for DatabaseFolderOperation {
_ => CSVFormat::Original,
};
FutureResult::new(async move {
- let content =
- String::from_utf8(bytes).map_err(|err| FlowyError::internal().with_context(err))?;
+ let content = tokio::task::spawn_blocking(move || {
+ String::from_utf8(bytes).map_err(|err| FlowyError::internal().with_context(err))
+ })
+ .await??;
+
database_manager
.import_csv(view_id, content, format)
.await?;
diff --git a/frontend/rust-lib/flowy-database2/Cargo.toml b/frontend/rust-lib/flowy-database2/Cargo.toml
index 2d6cc2ec97..930cad50d2 100644
--- a/frontend/rust-lib/flowy-database2/Cargo.toml
+++ b/frontend/rust-lib/flowy-database2/Cargo.toml
@@ -40,7 +40,7 @@ futures.workspace = true
dashmap = "5"
anyhow.workspace = true
async-stream = "0.3.4"
-rayon = "1.6.1"
+rayon = "1.9.0"
nanoid = "0.4.0"
async-trait.workspace = true
chrono-tz = "0.8.2"
diff --git a/frontend/rust-lib/flowy-database2/src/entities/database_entities.rs b/frontend/rust-lib/flowy-database2/src/entities/database_entities.rs
index 9002ca295e..2afcd41e05 100644
--- a/frontend/rust-lib/flowy-database2/src/entities/database_entities.rs
+++ b/frontend/rust-lib/flowy-database2/src/entities/database_entities.rs
@@ -1,6 +1,6 @@
use collab::core::collab_state::SyncState;
use collab_database::rows::RowId;
-use collab_database::user::DatabaseViewTracker;
+use collab_database::user::DatabaseMeta;
use collab_database::views::DatabaseLayout;
use flowy_derive::ProtoBuf;
@@ -208,8 +208,8 @@ pub struct DatabaseDescriptionPB {
pub database_id: String,
}
-impl From<DatabaseViewTracker> for DatabaseDescriptionPB {
- fn from(data: DatabaseViewTracker) -> Self {
+impl From<DatabaseMeta> for DatabaseDescriptionPB {
+ fn from(data: DatabaseMeta) -> Self {
Self {
database_id: data.database_id,
}
diff --git a/frontend/rust-lib/flowy-database2/src/manager.rs b/frontend/rust-lib/flowy-database2/src/manager.rs
index 7dc17d9e8d..5a21e3b18a 100644
--- a/frontend/rust-lib/flowy-database2/src/manager.rs
+++ b/frontend/rust-lib/flowy-database2/src/manager.rs
@@ -168,7 +168,7 @@ impl DatabaseManager {
let mut items = vec![];
if let Ok(wdb) = self.get_workspace_database().await {
items = wdb
- .get_all_databases()
+ .get_all_database_meta()
.into_iter()
.map(DatabaseDescriptionPB::from)
.collect();
diff --git a/frontend/rust-lib/flowy-database2/tests/database/share_test/export_test.rs b/frontend/rust-lib/flowy-database2/tests/database/share_test/export_test.rs
index 34ef732f39..e7c227f26d 100644
--- a/frontend/rust-lib/flowy-database2/tests/database/share_test/export_test.rs
+++ b/frontend/rust-lib/flowy-database2/tests/database/share_test/export_test.rs
@@ -124,6 +124,9 @@ async fn history_database_import_test() {
"{""data"":""AE"",""field_type"":0}","{""data"":""5"",""field_type"":1}","{""data"":""1671938394"",""field_type"":2}","{""data"":""wQpG"",""field_type"":3}","{""data"":"""",""field_type"":4}","{""data"":""Yes"",""field_type"":5}","{""data"":"""",""field_type"":6}","{""data"":"""",""field_type"":7}"
"#;
let result = test.import(csv.to_string(), format).await;
+ // TODO(nathan): remove this sleep
+ // workaround for the rows that are created asynchronously
+ tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
let database = test.get_database(&result.database_id).await.unwrap();
let fields = database.get_fields(&result.view_id, None);
diff --git a/frontend/rust-lib/flowy-folder/src/entities/view.rs b/frontend/rust-lib/flowy-folder/src/entities/view.rs
index 7b756163a6..508d68989c 100644
--- a/frontend/rust-lib/flowy-folder/src/entities/view.rs
+++ b/frontend/rust-lib/flowy-folder/src/entities/view.rs
@@ -59,14 +59,14 @@ pub struct ViewPB {
pub is_favorite: bool,
}
-pub fn view_pb_without_child_views(view: Arc<View>) -> ViewPB {
+pub fn view_pb_without_child_views(view: View) -> ViewPB {
ViewPB {
- id: view.id.clone(),
- parent_view_id: view.parent_view_id.clone(),
- name: view.name.clone(),
+ id: view.id,
+ parent_view_id: view.parent_view_id,
+ name: view.name,
create_time: view.created_at,
child_views: Default::default(),
- layout: view.layout.clone().into(),
+ layout: view.layout.into(),
icon: view.icon.clone().map(|icon| icon.into()),
is_favorite: view.is_favorite,
}
@@ -81,7 +81,7 @@ pub fn view_pb_with_child_views(view: Arc<View>, child_views: Vec<Arc<View>>) ->
create_time: view.created_at,
child_views: child_views
.into_iter()
- .map(view_pb_without_child_views)
+ .map(|view| view_pb_without_child_views(view.as_ref().clone()))
.collect(),
layout: view.layout.clone().into(),
icon: view.icon.clone().map(|icon| icon.into()),
diff --git a/frontend/rust-lib/flowy-folder/src/event_handler.rs b/frontend/rust-lib/flowy-folder/src/event_handler.rs
index a5412decc6..83530dc547 100644
--- a/frontend/rust-lib/flowy-folder/src/event_handler.rs
+++ b/frontend/rust-lib/flowy-folder/src/event_handler.rs
@@ -28,7 +28,7 @@ pub(crate) async fn create_workspace_handler(
.get_views_belong_to(&workspace.id)
.await?
.into_iter()
- .map(view_pb_without_child_views)
+ .map(|view| view_pb_without_child_views(view.as_ref().clone()))
.collect::<Vec<ViewPB>>();
data_result_ok(WorkspacePB {
id: workspace.id,
@@ -85,7 +85,7 @@ pub(crate) async fn create_view_handler(
if set_as_current {
let _ = folder.set_current_view(&view.id).await;
}
- data_result_ok(view_pb_without_child_views(Arc::new(view)))
+ data_result_ok(view_pb_without_child_views(view))
}
pub(crate) async fn create_orphan_view_handler(
@@ -99,7 +99,7 @@ pub(crate) async fn create_orphan_view_handler(
if set_as_current {
let _ = folder.set_current_view(&view.id).await;
}
- data_result_ok(view_pb_without_child_views(Arc::new(view)))
+ data_result_ok(view_pb_without_child_views(view))
}
#[tracing::instrument(level = "debug", skip(data, folder), err)]
@@ -313,11 +313,12 @@ pub(crate) async fn delete_all_trash_handler(
pub(crate) async fn import_data_handler(
data: AFPluginData<ImportPB>,
folder: AFPluginState<Weak<FolderManager>>,
-) -> Result<(), FlowyError> {
+) -> DataResult<ViewPB, FlowyError> {
let folder = upgrade_folder(folder)?;
let params: ImportParams = data.into_inner().try_into()?;
- folder.import(params).await?;
- Ok(())
+ let view = folder.import(params).await?;
+ let view_pb = view_pb_without_child_views(view);
+ data_result_ok(view_pb)
}
#[tracing::instrument(level = "debug", skip(folder), err)]
diff --git a/frontend/rust-lib/flowy-folder/src/event_map.rs b/frontend/rust-lib/flowy-folder/src/event_map.rs
index 6a4d1faa20..a3f8d2b657 100644
--- a/frontend/rust-lib/flowy-folder/src/event_map.rs
+++ b/frontend/rust-lib/flowy-folder/src/event_map.rs
@@ -124,7 +124,7 @@ pub enum FolderEvent {
#[event()]
PermanentlyDeleteAllTrashItem = 27,
- #[event(input = "ImportPB")]
+ #[event(input = "ImportPB", output = "ViewPB")]
ImportData = 30,
#[event(input = "WorkspaceIdPB", output = "RepeatedFolderSnapshotPB")]
diff --git a/frontend/rust-lib/flowy-folder/src/manager.rs b/frontend/rust-lib/flowy-folder/src/manager.rs
index 04c1b2b4fc..b4d480ea38 100644
--- a/frontend/rust-lib/flowy-folder/src/manager.rs
+++ b/frontend/rust-lib/flowy-folder/src/manager.rs
@@ -374,7 +374,7 @@ impl FolderManager {
.views
.get_views_belong_to(&workspace.id)
.into_iter()
- .map(view_pb_without_child_views)
+ .map(|view| view_pb_without_child_views(view.as_ref().clone()))
.collect::<Vec<ViewPB>>();
WorkspacePB {
@@ -556,7 +556,7 @@ impl FolderManager {
.send();
notify_child_views_changed(
- view_pb_without_child_views(view),
+ view_pb_without_child_views(view.as_ref().clone()),
ChildViewChangeReason::Delete,
);
}
@@ -573,7 +573,7 @@ impl FolderManager {
let favorite_descendant_views: Vec<ViewPB> = all_descendant_views
.iter()
.filter(|view| view.is_favorite)
- .map(|view| view_pb_without_child_views(view.clone()))
+ .map(|view| view_pb_without_child_views(view.as_ref().clone()))
.collect();
if !favorite_descendant_views.is_empty() {
diff --git a/frontend/rust-lib/flowy-folder/src/manager_observer.rs b/frontend/rust-lib/flowy-folder/src/manager_observer.rs
index e37f20f31b..ab08010613 100644
--- a/frontend/rust-lib/flowy-folder/src/manager_observer.rs
+++ b/frontend/rust-lib/flowy-folder/src/manager_observer.rs
@@ -32,7 +32,7 @@ pub(crate) fn subscribe_folder_view_changed(
match value {
ViewChange::DidCreateView { view } => {
notify_child_views_changed(
- view_pb_without_child_views(Arc::new(view.clone())),
+ view_pb_without_child_views(view.clone()),
ChildViewChangeReason::Create,
);
notify_parent_view_did_change(folder.clone(), vec![view.parent_view_id]);
@@ -40,7 +40,7 @@ pub(crate) fn subscribe_folder_view_changed(
ViewChange::DidDeleteView { views } => {
for view in views {
notify_child_views_changed(
- view_pb_without_child_views(view),
+ view_pb_without_child_views(view.as_ref().clone()),
ChildViewChangeReason::Delete,
);
}
@@ -48,7 +48,7 @@ pub(crate) fn subscribe_folder_view_changed(
ViewChange::DidUpdate { view } => {
notify_view_did_change(view.clone());
notify_child_views_changed(
- view_pb_without_child_views(Arc::new(view.clone())),
+ view_pb_without_child_views(view.clone()),
ChildViewChangeReason::Update,
);
notify_parent_view_did_change(folder.clone(), vec![view.parent_view_id.clone()]);
@@ -190,8 +190,9 @@ pub(crate) fn notify_did_update_workspace(workspace_id: &str, folder: &Folder) {
}
fn notify_view_did_change(view: View) -> Option<()> {
- let view_pb = view_pb_without_child_views(Arc::new(view.clone()));
- send_notification(&view.id, FolderNotification::DidUpdateView)
+ let view_id = view.id.clone();
+ let view_pb = view_pb_without_child_views(view);
+ send_notification(&view_id, FolderNotification::DidUpdateView)
.payload(view_pb)
.send();
None
diff --git a/frontend/rust-lib/flowy-user/src/anon_user/migrate_anon_user_collab.rs b/frontend/rust-lib/flowy-user/src/anon_user/migrate_anon_user_collab.rs
index 18a5de7ff0..4761b55b73 100644
--- a/frontend/rust-lib/flowy-user/src/anon_user/migrate_anon_user_collab.rs
+++ b/frontend/rust-lib/flowy-user/src/anon_user/migrate_anon_user_collab.rs
@@ -10,7 +10,7 @@ use collab_database::database::{
is_database_collab, mut_database_views_with_collab, reset_inline_view_id,
};
use collab_database::rows::{database_row_document_id_from_row_id, mut_row_with_collab, RowId};
-use collab_database::user::DatabaseViewTrackerList;
+use collab_database::user::DatabaseMetaList;
use collab_folder::{Folder, UserId};
use collab_plugins::local_storage::kv::KVTransactionDB;
use parking_lot::{Mutex, RwLock};
@@ -163,9 +163,9 @@ where
let new_uid = new_user_session.user_id;
let new_object_id = &new_user_session.user_workspace.workspace_database_object_id;
- let array = DatabaseViewTrackerList::from_collab(&database_with_views_collab);
- for database_view_tracker in array.get_all_database_tracker() {
- array.update_database(&database_view_tracker.database_id, |update| {
+ let array = DatabaseMetaList::from_collab(&database_with_views_collab);
+ for database_metas in array.get_all_database_meta() {
+ array.update_database(&database_metas.database_id, |update| {
let new_linked_views = update
.linked_views
.iter()
diff --git a/frontend/rust-lib/flowy-user/src/anon_user/sync_supabase_user_collab.rs b/frontend/rust-lib/flowy-user/src/anon_user/sync_supabase_user_collab.rs
index 6387ec7ba5..5fb01aae5a 100644
--- a/frontend/rust-lib/flowy-user/src/anon_user/sync_supabase_user_collab.rs
+++ b/frontend/rust-lib/flowy-user/src/anon_user/sync_supabase_user_collab.rs
@@ -8,7 +8,7 @@ use collab::core::collab::MutexCollab;
use collab::preclude::Collab;
use collab_database::database::get_database_row_ids;
use collab_database::rows::database_row_document_id_from_row_id;
-use collab_database::user::{get_all_database_view_trackers, DatabaseViewTracker};
+use collab_database::user::{get_all_database_meta, DatabaseMeta};
use collab_entity::{CollabObject, CollabType};
use collab_folder::{Folder, View, ViewLayout};
use collab_plugins::local_storage::kv::KVTransactionDB;
@@ -75,7 +75,7 @@ pub async fn sync_supabase_user_data_to_cloud(
fn sync_view(
uid: i64,
folder: Arc<MutexFolder>,
- database_records: Vec<Arc<DatabaseViewTracker>>,
+ database_records: Vec<Arc<DatabaseMeta>>,
workspace_id: String,
device_id: String,
view: Arc<View>,
@@ -297,7 +297,7 @@ async fn sync_database_views(
database_views_aggregate_id: &str,
collab_db: &Arc<CollabKVDB>,
user_service: Arc<dyn UserCloudService>,
-) -> Vec<Arc<DatabaseViewTracker>> {
+) -> Vec<Arc<DatabaseMeta>> {
let collab_object = CollabObject::new(
uid,
database_views_aggregate_id.to_string(),
@@ -317,7 +317,7 @@ async fn sync_database_views(
})
.map(|_| {
(
- get_all_database_view_trackers(&collab),
+ get_all_database_meta(&collab),
collab.encode_collab_v1().doc_state,
)
})
@@ -357,7 +357,7 @@ fn collab_type_from_view_layout(view_layout: &ViewLayout) -> CollabType {
fn object_id_from_view(
view: &Arc<View>,
- database_records: &[Arc<DatabaseViewTracker>],
+ database_records: &[Arc<DatabaseMeta>],
) -> Result<String, Error> {
if view.layout.is_database() {
match database_records
diff --git a/frontend/rust-lib/flowy-user/src/services/data_import/appflowy_data_import.rs b/frontend/rust-lib/flowy-user/src/services/data_import/appflowy_data_import.rs
index 6783a24b04..e48df18254 100644
--- a/frontend/rust-lib/flowy-user/src/services/data_import/appflowy_data_import.rs
+++ b/frontend/rust-lib/flowy-user/src/services/data_import/appflowy_data_import.rs
@@ -14,7 +14,7 @@ use collab_database::database::{
is_database_collab, mut_database_views_with_collab, reset_inline_view_id,
};
use collab_database::rows::{database_row_document_id_from_row_id, mut_row_with_collab, RowId};
-use collab_database::user::DatabaseViewTrackerList;
+use collab_database::user::DatabaseMetaList;
use collab_document::document_data::default_document_collab_data;
use collab_entity::CollabType;
use collab_folder::{Folder, UserId, View, ViewIdentifier, ViewLayout};
@@ -280,11 +280,11 @@ where
)
})?;
- let array = DatabaseViewTrackerList::from_collab(&database_view_tracker_collab);
- for database_view_tracker in array.get_all_database_tracker() {
+ let array = DatabaseMetaList::from_collab(&database_view_tracker_collab);
+ for database_metas in array.get_all_database_meta() {
database_view_ids_by_database_id.insert(
- old_to_new_id_map.renew_id(&database_view_tracker.database_id),
- database_view_tracker
+ old_to_new_id_map.renew_id(&database_metas.database_id),
+ database_metas
.linked_views
.into_iter()
.map(|view_id| old_to_new_id_map.renew_id(&view_id))