Skip to content

Bring projects, wikis, and teams together with AI. AppFlowy is the AI collaborative workspace where you achieve more without losing control of your data. The leading open source Notion alternative.

feat: support local ai embedding, local ai search, local document content search (#7839)

Nathan.fooo612d65232048
diff --git a/.github/workflows/android_ci.yaml.bak b/.github/workflows/android_ci.yaml.bak
index 81e132cbf8..27e680eada 100644
--- a/.github/workflows/android_ci.yaml.bak
+++ b/.github/workflows/android_ci.yaml.bak
@@ -19,7 +19,7 @@ on:
env:
CARGO_TERM_COLOR: always
FLUTTER_VERSION: "3.27.4"
- RUST_TOOLCHAIN: "1.81.0"
+ RUST_TOOLCHAIN: "1.84.0"
CARGO_MAKE_VERSION: "0.37.18"
CLOUD_VERSION: 0.6.54-amd64
diff --git a/.github/workflows/flutter_ci.yaml b/.github/workflows/flutter_ci.yaml
index b939a63aaf..07cfd9933d 100644
--- a/.github/workflows/flutter_ci.yaml
+++ b/.github/workflows/flutter_ci.yaml
@@ -26,7 +26,7 @@ on:
env:
CARGO_TERM_COLOR: always
FLUTTER_VERSION: "3.27.4"
- RUST_TOOLCHAIN: "1.81.0"
+ RUST_TOOLCHAIN: "1.84.0"
CARGO_MAKE_VERSION: "0.37.18"
CLOUD_VERSION: 0.9.37-amd64
diff --git a/.github/workflows/ios_ci.yaml b/.github/workflows/ios_ci.yaml
index ab604c23ce..4459aac631 100644
--- a/.github/workflows/ios_ci.yaml
+++ b/.github/workflows/ios_ci.yaml
@@ -19,7 +19,7 @@ on:
env:
FLUTTER_VERSION: "3.27.4"
- RUST_TOOLCHAIN: "1.81.0"
+ RUST_TOOLCHAIN: "1.84.0"
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 422649a25e..dbf20499ab 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -7,7 +7,7 @@ on:
env:
FLUTTER_VERSION: "3.27.4"
- RUST_TOOLCHAIN: "1.81.0"
+ RUST_TOOLCHAIN: "1.84.0"
jobs:
create-release:
@@ -232,10 +232,10 @@ jobs:
matrix:
job:
- {
- targets: "aarch64-apple-darwin,x86_64-apple-darwin",
- os: macos-latest,
- extra-build-args: "",
- }
+ targets: "aarch64-apple-darwin,x86_64-apple-darwin",
+ os: macos-latest,
+ extra-build-args: "",
+ }
steps:
- name: Checkout source code
uses: actions/checkout@v4
@@ -336,12 +336,12 @@ jobs:
matrix:
job:
- {
- arch: x86_64,
- target: x86_64-unknown-linux-gnu,
- os: ubuntu-22.04,
- extra-build-args: "",
- flutter_profile: production-linux-x86_64,
- }
+ arch: x86_64,
+ target: x86_64-unknown-linux-gnu,
+ os: ubuntu-22.04,
+ extra-build-args: "",
+ flutter_profile: production-linux-x86_64,
+ }
steps:
- name: Checkout source code
uses: actions/checkout@v4
diff --git a/.github/workflows/rust_ci.yaml b/.github/workflows/rust_ci.yaml
index 36c2e82064..4ff78c689d 100644
--- a/.github/workflows/rust_ci.yaml
+++ b/.github/workflows/rust_ci.yaml
@@ -19,7 +19,7 @@ on:
env:
CARGO_TERM_COLOR: always
CLOUD_VERSION: 0.8.3-amd64
- RUST_TOOLCHAIN: "1.81.0"
+ RUST_TOOLCHAIN: "1.84.0"
jobs:
ubuntu-job:
diff --git a/.github/workflows/rust_coverage.yml b/.github/workflows/rust_coverage.yml
index 53a5f66748..a22e81e59a 100644
--- a/.github/workflows/rust_coverage.yml
+++ b/.github/workflows/rust_coverage.yml
@@ -11,7 +11,7 @@ on:
env:
CARGO_TERM_COLOR: always
FLUTTER_VERSION: "3.27.4"
- RUST_TOOLCHAIN: "1.81.0"
+ RUST_TOOLCHAIN: "1.84.0"
jobs:
tests:
diff --git a/frontend/Makefile.toml b/frontend/Makefile.toml
index 01a756a173..84562624e3 100644
--- a/frontend/Makefile.toml
+++ b/frontend/Makefile.toml
@@ -1,16 +1,16 @@
#https://github.com/sagiegurari/cargo-make
extend = [
- { path = "scripts/makefile/desktop.toml" },
- { path = "scripts/makefile/mobile.toml" },
- { path = "scripts/makefile/protobuf.toml" },
- { path = "scripts/makefile/tests.toml" },
- { path = "scripts/makefile/docker.toml" },
- { path = "scripts/makefile/env.toml" },
- { path = "scripts/makefile/flutter.toml" },
- { path = "scripts/makefile/tool.toml" },
- { path = "scripts/makefile/tauri.toml" },
- { path = "scripts/makefile/web.toml" },
+ { path = "scripts/makefile/desktop.toml" },
+ { path = "scripts/makefile/mobile.toml" },
+ { path = "scripts/makefile/protobuf.toml" },
+ { path = "scripts/makefile/tests.toml" },
+ { path = "scripts/makefile/docker.toml" },
+ { path = "scripts/makefile/env.toml" },
+ { path = "scripts/makefile/flutter.toml" },
+ { path = "scripts/makefile/tool.toml" },
+ { path = "scripts/makefile/tauri.toml" },
+ { path = "scripts/makefile/web.toml" },
]
[config]
@@ -228,30 +228,31 @@ script = ['''
echo APP_ENVIRONMENT: ${APP_ENVIRONMENT}
echo BUILD_ARCHS: ${BUILD_ARCHS}
echo BUILD_VERSION: ${BUILD_VERSION}
+ echo RUST_VERSION: "$(rustc --version)"
''']
script_runner = "@shell"
[tasks.setup-crate-type]
private = true
script = [
- """
- toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
- val = replace ${toml} "staticlib" ${CRATE_TYPE}
- result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
- assert ${result}
- """,
+ """
+ toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
+ val = replace ${toml} "staticlib" ${CRATE_TYPE}
+ result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
+ assert ${result}
+ """,
]
script_runner = "@duckscript"
[tasks.restore-crate-type]
private = true
script = [
- """
- toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
- val = replace ${toml} ${CRATE_TYPE} "staticlib"
- result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
- assert ${result}
- """,
+ """
+ toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
+ val = replace ${toml} ${CRATE_TYPE} "staticlib"
+ result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
+ assert ${result}
+ """,
]
script_runner = "@duckscript"
@@ -279,24 +280,24 @@ TEST_COMPILE_TARGET = "x86_64-pc-windows-msvc"
[tasks.setup-test-crate-type]
private = true
script = [
- """
- toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
- val = replace ${toml} "staticlib" ${TEST_CRATE_TYPE}
- result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
- assert ${result}
- """,
+ """
+ toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
+ val = replace ${toml} "staticlib" ${TEST_CRATE_TYPE}
+ result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
+ assert ${result}
+ """,
]
script_runner = "@duckscript"
[tasks.restore-test-crate-type]
private = true
script = [
- """
- toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
- val = replace ${toml} ${TEST_CRATE_TYPE} "staticlib"
- result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
- assert ${result}
- """,
+ """
+ toml = readfile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml
+ val = replace ${toml} ${TEST_CRATE_TYPE} "staticlib"
+ result = writefile ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/rust-lib/${CARGO_MAKE_CRATE_NAME}/Cargo.toml ${val}
+ assert ${result}
+ """,
]
script_runner = "@duckscript"
diff --git a/frontend/appflowy_flutter/lib/workspace/application/command_palette/command_palette_bloc.dart b/frontend/appflowy_flutter/lib/workspace/application/command_palette/command_palette_bloc.dart
index 9cee2cd9e1..116598d63f 100644
--- a/frontend/appflowy_flutter/lib/workspace/application/command_palette/command_palette_bloc.dart
+++ b/frontend/appflowy_flutter/lib/workspace/application/command_palette/command_palette_bloc.dart
@@ -49,7 +49,6 @@ class CommandPaletteBloc
final TrashService _trashService = TrashService();
final TrashListener _trashListener = TrashListener();
String? _activeQuery;
- String? _workspaceId;
@override
Future<void> close() {
@@ -115,7 +114,6 @@ class CommandPaletteBloc
unawaited(
SearchBackendService.performSearch(
event.search,
- workspaceId: _workspaceId,
).then(
(result) => result.fold(
(stream) {
@@ -161,6 +159,7 @@ class CommandPaletteBloc
onServerItems: (items, searchId, searching, generatingAIOverview) =>
_handleResultsUpdate(
searchId: searchId,
+ summaries: [], // when got server search result, summaries should be empty
serverItems: items,
searching: searching,
generatingAIOverview: generatingAIOverview,
@@ -263,7 +262,6 @@ class CommandPaletteBloc
_WorkspaceChanged event,
Emitter<CommandPaletteState> emit,
) {
- _workspaceId = event.workspaceId;
emit(
state.copyWith(
query: '',
diff --git a/frontend/appflowy_flutter/lib/workspace/application/command_palette/search_service.dart b/frontend/appflowy_flutter/lib/workspace/application/command_palette/search_service.dart
index 89e5b604f8..6622854a98 100644
--- a/frontend/appflowy_flutter/lib/workspace/application/command_palette/search_service.dart
+++ b/frontend/appflowy_flutter/lib/workspace/application/command_palette/search_service.dart
@@ -8,28 +8,23 @@ import 'package:appflowy_backend/protobuf/flowy-error/errors.pb.dart';
import 'package:appflowy_backend/protobuf/flowy-search/notification.pb.dart';
import 'package:appflowy_backend/protobuf/flowy-search/query.pb.dart';
import 'package:appflowy_backend/protobuf/flowy-search/result.pb.dart';
-import 'package:appflowy_backend/protobuf/flowy-search/search_filter.pb.dart';
import 'package:appflowy_result/appflowy_result.dart';
-import 'package:nanoid/nanoid.dart';
import 'package:fixnum/fixnum.dart';
class SearchBackendService {
static Future<FlowyResult<SearchResponseStream, FlowyError>> performSearch(
- String keyword, {
- String? workspaceId,
- }) async {
- final searchId = nanoid(6);
+ String keyword,
+ ) async {
+ final searchId = DateTime.now().millisecondsSinceEpoch.toString();
final stream = SearchResponseStream(searchId: searchId);
- final filter = SearchFilterPB(workspaceId: workspaceId);
final request = SearchQueryPB(
search: keyword,
- filter: filter,
searchId: searchId,
streamPort: Int64(stream.nativePort),
);
- unawaited(SearchEventSearch(request).send());
+ unawaited(SearchEventStreamSearch(request).send());
return FlowyResult.success(stream);
}
}
diff --git a/frontend/appflowy_flutter/macos/Podfile.lock b/frontend/appflowy_flutter/macos/Podfile.lock
index a1eb7c5d14..6b4b65908c 100644
--- a/frontend/appflowy_flutter/macos/Podfile.lock
+++ b/frontend/appflowy_flutter/macos/Podfile.lock
@@ -144,34 +144,34 @@ EXTERNAL SOURCES:
:path: Flutter/ephemeral/.symlinks/plugins/window_manager/macos
SPEC CHECKSUMS:
- app_links: 9028728e32c83a0831d9db8cf91c526d16cc5468
- appflowy_backend: 464aeb3e5c6966a41641a2111e5ead72ce2695f7
- auto_updater_macos: 3a42f1a06be6981f1a18be37e6e7bf86aa732118
- bitsdojo_window_macos: 7959fb0ca65a3ccda30095c181ecb856fae48ea9
- connectivity_plus: e74b9f74717d2d99d45751750e266e55912baeb5
- desktop_drop: e0b672a7d84c0a6cbc378595e82cdb15f2970a43
- device_info_plus: 4fb280989f669696856f8b129e4a5e3cd6c48f76
- file_selector_macos: 6280b52b459ae6c590af5d78fc35c7267a3c4b31
- flowy_infra_ui: 8760ff42a789de40bf5007a5f176b454722a341e
+ app_links: 10e0a0ab602ffaf34d142cd4862f29d34b303b2a
+ appflowy_backend: 865496343de667fc8c600e04b9fd05234e130cf9
+ auto_updater_macos: 3e3462c418fe4e731917eacd8d28eef7af84086d
+ bitsdojo_window_macos: 44e3b8fe3dd463820e0321f6256c5b1c16bb6a00
+ connectivity_plus: 18d3c32514c886e046de60e9c13895109866c747
+ desktop_drop: 69eeff437544aa619c8db7f4481b3a65f7696898
+ device_info_plus: 1b14eed9bf95428983aed283a8d51cce3d8c4215
+ file_selector_macos: cc3858c981fe6889f364731200d6232dac1d812d
+ flowy_infra_ui: 03301a39ad118771adbf051a664265c61c507f38
FlutterMacOS: 8f6f14fa908a6fb3fba0cd85dbd81ec4b251fb24
HotKey: 400beb7caa29054ea8d864c96f5ba7e5b4852277
- hotkey_manager: b443f35f4d772162937aa73fd8995e579f8ac4e2
- irondash_engine_context: 893c7d96d20ce361d7e996f39d360c4c2f9869ba
- local_notifier: ebf072651e35ae5e47280ad52e2707375cb2ae4e
- package_info_plus: f0052d280d17aa382b932f399edf32507174e870
- path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564
+ hotkey_manager: c32bf0bfe8f934b7bc17ab4ad5c4c142960b023c
+ irondash_engine_context: da62996ee25616d2f01bbeb85dc115d813359478
+ local_notifier: e9506bc66fc70311e8bc7291fb70f743c081e4ff
+ package_info_plus: 12f1c5c2cfe8727ca46cbd0b26677728972d9a5b
+ path_provider_foundation: 2b6b4c569c0fb62ec74538f866245ac84301af46
ReachabilitySwift: 32793e867593cfc1177f5d16491e3a197d2fccda
- screen_retriever_macos: 452e51764a9e1cdb74b3c541238795849f21557f
+ screen_retriever_macos: 776e0fa5d42c6163d2bf772d22478df4b302b161
Sentry: da60d980b197a46db0b35ea12cb8f39af48d8854
- sentry_flutter: 27892878729f42701297c628eb90e7c6529f3684
- share_plus: 510bf0af1a42cd602274b4629920c9649c52f4cc
- shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7
+ sentry_flutter: 2df8b0aab7e4aba81261c230cbea31c82a62dd1b
+ share_plus: 1fa619de8392a4398bfaf176d441853922614e89
+ shared_preferences_foundation: fcdcbc04712aee1108ac7fda236f363274528f78
Sparkle: 9c328bdcfbcaf8f030c4b678eadfd0fcb03822d8
- sqflite_darwin: 20b2a3a3b70e43edae938624ce550a3cbf66a3d0
- super_native_extensions: c2795d6d9aedf4a79fae25cb6160b71b50549189
- url_launcher_macos: 0fba8ddabfc33ce0a9afe7c5fef5aab3d8d2d673
- webview_flutter_wkwebview: 44d4dee7d7056d5ad185d25b38404436d56c547c
- window_manager: e8d0b1431ab6c454f2b5c9ae26004bbfa43469aa
+ sqflite_darwin: 5a7236e3b501866c1c9befc6771dfd73ffb8702d
+ super_native_extensions: 85efee3a7495b46b04befcfc86ed12069264ebf3
+ url_launcher_macos: c82c93949963e55b228a30115bd219499a6fe404
+ webview_flutter_wkwebview: 0982481e3d9c78fd5c6f62a002fcd24fc791f1e4
+ window_manager: 990c8e348c4da2a93b81da638245d40554ec9436
PODFILE CHECKSUM: 0532f3f001ca3110b8be345d6491fff690e95823
diff --git a/frontend/rust-lib/Cargo.lock b/frontend/rust-lib/Cargo.lock
index 6521d3999b..a145257de5 100644
--- a/frontend/rust-lib/Cargo.lock
+++ b/frontend/rust-lib/Cargo.lock
@@ -1,6 +1,6 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
-version = 3
+version = 4
[[package]]
name = "accessory"
@@ -20,7 +20,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a"
dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.9.0",
"bytes",
"futures-core",
"futures-sink",
@@ -57,9 +57,9 @@ dependencies = [
"actix-service",
"actix-tls",
"actix-utils",
- "ahash 0.8.6",
+ "ahash 0.8.11",
"base64 0.21.5",
- "bitflags 2.4.0",
+ "bitflags 2.9.0",
"brotli",
"bytes",
"bytestring",
@@ -193,7 +193,7 @@ dependencies = [
"actix-tls",
"actix-utils",
"actix-web-codegen",
- "ahash 0.8.6",
+ "ahash 0.8.11",
"bytes",
"bytestring",
"cfg-if",
@@ -242,7 +242,7 @@ dependencies = [
"actix-utils",
"actix-web",
"actix-web-lab-derive",
- "ahash 0.8.6",
+ "ahash 0.8.11",
"arc-swap",
"async-trait",
"bytes",
@@ -418,15 +418,15 @@ dependencies = [
[[package]]
name = "ahash"
-version = "0.8.6"
+version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
+checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
dependencies = [
"cfg-if",
"getrandom 0.2.10",
"once_cell",
"version_check",
- "zerocopy",
+ "zerocopy 0.7.35",
]
[[package]]
@@ -493,7 +493,7 @@ checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
[[package]]
name = "app-error"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"anyhow",
"bincode",
@@ -513,7 +513,7 @@ dependencies = [
[[package]]
name = "appflowy-ai-client"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"anyhow",
"bytes",
@@ -697,6 +697,18 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112ef6b3f6cb3cb6fc5b6b494ef7a848492cff1ab0ef4de10b0f7d572861c905"
+[[package]]
+name = "auto_enums"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c170965892137a3a9aeb000b4524aa3cc022a310e709d848b6e1cdce4ab4781"
+dependencies = [
+ "derive_utils",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.94",
+]
+
[[package]]
name = "autocfg"
version = "1.1.0"
@@ -816,7 +828,7 @@ version = "0.69.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.9.0",
"cexpr",
"clang-sys",
"itertools 0.12.1",
@@ -853,9 +865,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
-version = "2.4.0"
+version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
+checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
[[package]]
name = "bitpacking"
@@ -1042,9 +1054,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.0.83"
+version = "1.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
+checksum = "e9e8aabfac534be767c909e0690571677d49f41bd8465ae876fe043d52ba5292"
dependencies = [
"jobserver",
"libc",
@@ -1159,7 +1171,7 @@ dependencies = [
[[package]]
name = "client-api"
version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"again",
"anyhow",
@@ -1214,7 +1226,7 @@ dependencies = [
[[package]]
name = "client-api-entity"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"collab-entity",
"collab-rt-entity",
@@ -1227,7 +1239,7 @@ dependencies = [
[[package]]
name = "client-websocket"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"futures-channel",
"futures-util",
@@ -1270,7 +1282,7 @@ dependencies = [
[[package]]
name = "collab"
version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3#f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe#b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe"
dependencies = [
"anyhow",
"arc-swap",
@@ -1295,7 +1307,7 @@ dependencies = [
[[package]]
name = "collab-database"
version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3#f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe#b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe"
dependencies = [
"anyhow",
"async-trait",
@@ -1321,7 +1333,7 @@ dependencies = [
"serde_json",
"serde_repr",
"sha2",
- "strum",
+ "strum 0.25.0",
"strum_macros 0.25.2",
"thiserror 1.0.64",
"tokio",
@@ -1335,7 +1347,7 @@ dependencies = [
[[package]]
name = "collab-document"
version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3#f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe#b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe"
dependencies = [
"anyhow",
"arc-swap",
@@ -1356,7 +1368,7 @@ dependencies = [
[[package]]
name = "collab-entity"
version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3#f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe#b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe"
dependencies = [
"anyhow",
"bytes",
@@ -1376,7 +1388,7 @@ dependencies = [
[[package]]
name = "collab-folder"
version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3#f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe#b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe"
dependencies = [
"anyhow",
"arc-swap",
@@ -1398,7 +1410,7 @@ dependencies = [
[[package]]
name = "collab-importer"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3#f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe#b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe"
dependencies = [
"anyhow",
"async-recursion",
@@ -1447,21 +1459,21 @@ dependencies = [
"collab-folder",
"collab-plugins",
"collab-user",
- "diesel",
+ "flowy-ai-pub",
"flowy-error",
- "flowy-sqlite",
"lib-infra",
"serde",
"serde_json",
"tokio",
"tracing",
+ "twox-hash",
"uuid",
]
[[package]]
name = "collab-plugins"
version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3#f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe#b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe"
dependencies = [
"anyhow",
"async-stream",
@@ -1477,7 +1489,6 @@ dependencies = [
"indexed_db_futures",
"js-sys",
"lazy_static",
- "rand 0.8.5",
"rocksdb",
"serde",
"serde_json",
@@ -1499,7 +1510,7 @@ dependencies = [
[[package]]
name = "collab-rt-entity"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"anyhow",
"bincode",
@@ -1521,7 +1532,7 @@ dependencies = [
[[package]]
name = "collab-rt-protocol"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"anyhow",
"async-trait",
@@ -1539,7 +1550,7 @@ dependencies = [
[[package]]
name = "collab-user"
version = "0.2.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3#f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe#b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe"
dependencies = [
"anyhow",
"collab",
@@ -1687,6 +1698,15 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
+[[package]]
+name = "core_maths"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30"
+dependencies = [
+ "libm",
+]
+
[[package]]
name = "cpufeatures"
version = "0.2.9"
@@ -1969,7 +1989,7 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308"
[[package]]
name = "database-entity"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"bincode",
"bytes",
@@ -2121,6 +2141,17 @@ dependencies = [
"syn 1.0.109",
]
+[[package]]
+name = "derive_utils"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.94",
+]
+
[[package]]
name = "deunicode"
version = "0.4.4"
@@ -2135,9 +2166,9 @@ checksum = "b6e854126756c496b8c81dec88f9a706b15b875c5849d4097a3854476b9fdf94"
[[package]]
name = "diesel"
-version = "2.1.4"
+version = "2.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62c6fcf842f17f8c78ecf7c81d75c5ce84436b41ee07e03f490fbb5f5a8731d8"
+checksum = "ff3e1edb1f37b4953dd5176916347289ed43d7119cc2e6c7c3f7849ff44ea506"
dependencies = [
"chrono",
"diesel_derives",
@@ -2149,11 +2180,12 @@ dependencies = [
[[package]]
name = "diesel_derives"
-version = "2.1.2"
+version = "2.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44"
+checksum = "68d4216021b3ea446fd2047f5c8f8fe6e98af34508a254a01e4d6bc1e844f84d"
dependencies = [
"diesel_table_macro_syntax",
+ "dsl_auto_type",
"proc-macro2",
"quote",
"syn 2.0.94",
@@ -2161,9 +2193,9 @@ dependencies = [
[[package]]
name = "diesel_migrations"
-version = "2.1.0"
+version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6036b3f0120c5961381b570ee20a02432d7e2d27ea60de9578799cf9156914ac"
+checksum = "8a73ce704bad4231f001bff3314d91dce4aba0770cee8b233991859abc15c1f6"
dependencies = [
"diesel",
"migrations_internals",
@@ -2172,9 +2204,9 @@ dependencies = [
[[package]]
name = "diesel_table_macro_syntax"
-version = "0.1.0"
+version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
+checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25"
dependencies = [
"syn 2.0.94",
]
@@ -2216,6 +2248,20 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea8a8b81cacc08888170eef4d13b775126db426d0b348bee9d18c2c1eaf123cf"
+[[package]]
+name = "dsl_auto_type"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "139ae9aca7527f85f26dd76483eb38533fd84bd571065da1739656ef71c5ff5b"
+dependencies = [
+ "darling 0.20.11",
+ "either",
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.94",
+]
+
[[package]]
name = "dtoa"
version = "1.0.9"
@@ -2245,9 +2291,9 @@ checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591"
[[package]]
name = "either"
-version = "1.9.0"
+version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "encode_unicode"
@@ -2339,7 +2385,8 @@ dependencies = [
"semver",
"serde",
"serde_json",
- "strum",
+ "serial_test",
+ "strum 0.25.0",
"tokio",
"tracing",
"uuid",
@@ -2400,6 +2447,18 @@ dependencies = [
"rand 0.8.5",
]
+[[package]]
+name = "fallible-iterator"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
+
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+
[[package]]
name = "fancy-regex"
version = "0.10.0"
@@ -2498,7 +2557,6 @@ dependencies = [
"arc-swap",
"base64 0.21.5",
"bytes",
- "collab-integrate",
"dashmap 6.0.1",
"flowy-ai-pub",
"flowy-codegen",
@@ -2506,6 +2564,7 @@ dependencies = [
"flowy-error",
"flowy-notification",
"flowy-sqlite",
+ "flowy-sqlite-vec",
"flowy-storage-pub",
"futures",
"futures-util",
@@ -2517,14 +2576,17 @@ dependencies = [
"pin-project",
"protobuf",
"reqwest 0.11.27",
+ "schemars",
"serde",
"serde_json",
"sha2",
"strum_macros 0.21.1",
+ "text-splitter",
"tokio",
"tokio-stream",
"tokio-util",
"tracing",
+ "twox-hash",
"uuid",
"validator 0.18.1",
]
@@ -2533,6 +2595,7 @@ dependencies = [
name = "flowy-ai-pub"
version = "0.1.0"
dependencies = [
+ "chrono",
"client-api",
"flowy-error",
"flowy-sqlite",
@@ -2540,6 +2603,7 @@ dependencies = [
"lib-infra",
"serde",
"serde_json",
+ "twox-hash",
"uuid",
]
@@ -2588,6 +2652,7 @@ dependencies = [
"bytes",
"client-api",
"collab",
+ "collab-document",
"collab-entity",
"collab-folder",
"collab-integrate",
@@ -2610,6 +2675,7 @@ dependencies = [
"flowy-server",
"flowy-server-pub",
"flowy-sqlite",
+ "flowy-sqlite-vec",
"flowy-storage",
"flowy-storage-pub",
"flowy-user",
@@ -2618,14 +2684,16 @@ dependencies = [
"lib-dispatch",
"lib-infra",
"lib-log",
+ "once_cell",
"semver",
"serde",
"serde_json",
- "serde_repr",
"sysinfo",
"tokio",
"tokio-stream",
+ "tokio-util",
"tracing",
+ "twox-hash",
"url",
"uuid",
]
@@ -2682,7 +2750,7 @@ dependencies = [
"serde",
"serde_json",
"serde_repr",
- "strum",
+ "strum 0.25.0",
"strum_macros 0.25.2",
"tokio",
"tokio-util",
@@ -2728,7 +2796,6 @@ dependencies = [
name = "flowy-document"
version = "0.1.0"
dependencies = [
- "anyhow",
"bytes",
"collab",
"collab-document",
@@ -2755,7 +2822,6 @@ dependencies = [
"strum_macros 0.21.1",
"tempfile",
"tokio",
- "tokio-stream",
"tracing",
"tracing-subscriber",
"uuid",
@@ -2881,10 +2947,12 @@ name = "flowy-search"
version = "0.1.0"
dependencies = [
"allo-isolate",
+ "arc-swap",
"async-stream",
"bytes",
"collab",
"collab-folder",
+ "dashmap 6.0.1",
"derive_builder 0.20.2",
"flowy-codegen",
"flowy-derive",
@@ -2898,7 +2966,6 @@ dependencies = [
"protobuf",
"serde",
"serde_json",
- "strsim 0.11.1",
"strum_macros 0.26.1",
"tantivy",
"tokio",
@@ -2932,7 +2999,6 @@ dependencies = [
"collab-database",
"collab-document",
"collab-entity",
- "collab-folder",
"collab-plugins",
"collab-user",
"flowy-ai",
@@ -2947,7 +3013,6 @@ dependencies = [
"flowy-storage",
"flowy-storage-pub",
"flowy-user-pub",
- "futures",
"futures-util",
"lazy_static",
"lib-infra",
@@ -2994,6 +3059,25 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "flowy-sqlite-vec"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "flowy-ai-pub",
+ "include_dir",
+ "rusqlite",
+ "rusqlite_migration",
+ "serde_json",
+ "sqlite-vec",
+ "tempfile",
+ "tokio",
+ "tracing",
+ "uuid",
+ "zerocopy 0.9.0-alpha.0",
+]
+
[[package]]
name = "flowy-storage"
version = "0.1.0"
@@ -3084,7 +3168,7 @@ dependencies = [
"semver",
"serde",
"serde_json",
- "strum",
+ "strum 0.25.0",
"strum_macros 0.25.2",
"tokio",
"tokio-stream",
@@ -3123,6 +3207,12 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+[[package]]
+name = "foldhash"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
[[package]]
name = "foreign-types"
version = "0.3.2"
@@ -3414,7 +3504,7 @@ dependencies = [
[[package]]
name = "gotrue"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"anyhow",
"getrandom 0.2.10",
@@ -3429,7 +3519,7 @@ dependencies = [
[[package]]
name = "gotrue-entity"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"app-error",
"jsonwebtoken",
@@ -3491,10 +3581,28 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
"allocator-api2",
]
+[[package]]
+name = "hashbrown"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
+dependencies = [
+ "foldhash",
+]
+
+[[package]]
+name = "hashlink"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
+dependencies = [
+ "hashbrown 0.15.2",
+]
+
[[package]]
name = "hdrhistogram"
version = "7.5.2"
@@ -3523,6 +3631,12 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
[[package]]
name = "hermit-abi"
version = "0.5.0"
@@ -3942,6 +4056,28 @@ dependencies = [
"syn 2.0.94",
]
+[[package]]
+name = "icu_segmenter"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a717725612346ffc2d7b42c94b820db6908048f39434504cb130e8b46256b0de"
+dependencies = [
+ "core_maths",
+ "displaydoc",
+ "icu_collections",
+ "icu_locid",
+ "icu_provider",
+ "icu_segmenter_data",
+ "utf8_iter",
+ "zerovec",
+]
+
+[[package]]
+name = "icu_segmenter_data"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1e52775179941363cc594e49ce99284d13d6948928d8e72c755f55e98caa1eb"
+
[[package]]
name = "ident_case"
version = "1.0.1"
@@ -4012,6 +4148,25 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2"
+[[package]]
+name = "include_dir"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd"
+dependencies = [
+ "include_dir_macros",
+]
+
+[[package]]
+name = "include_dir_macros"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75"
+dependencies = [
+ "proc-macro2",
+ "quote",
+]
+
[[package]]
name = "indexed_db_futures"
version = "0.4.2"
@@ -4054,7 +4209,7 @@ dependencies = [
[[package]]
name = "infra"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"anyhow",
"bytes",
@@ -4158,9 +4313,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
[[package]]
name = "jobserver"
-version = "0.1.26"
+version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
+checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
dependencies = [
"libc",
]
@@ -4345,9 +4500,9 @@ dependencies = [
[[package]]
name = "libsqlite3-sys"
-version = "0.27.0"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"
+checksum = "947e6816f7825b2b45027c2c32e7085da9934defa535de4a6a46b10a4d5257fa"
dependencies = [
"cc",
"pkg-config",
@@ -4630,19 +4785,19 @@ dependencies = [
[[package]]
name = "migrations_internals"
-version = "2.1.0"
+version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f23f71580015254b020e856feac3df5878c2c7a8812297edd6c0a485ac9dada"
+checksum = "fd01039851e82f8799046eabbb354056283fb265c8ec0996af940f4e85a380ff"
dependencies = [
"serde",
- "toml 0.7.8",
+ "toml 0.8.12",
]
[[package]]
name = "migrations_macros"
-version = "2.1.0"
+version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cce3325ac70e67bbab5bd837a31cae01f1a6db64e0e744a33cb03a543469ef08"
+checksum = "ffb161cc72176cb37aa47f1fc520d3ef02263d67d661f44f05d05a079e1237fd"
dependencies = [
"migrations_internals",
"proc-macro2",
@@ -4789,7 +4944,7 @@ version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d"
dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.9.0",
"crossbeam-channel",
"filetime",
"fsevent-sys",
@@ -4929,7 +5084,7 @@ version = "0.10.62"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671"
dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.9.0",
"cfg-if",
"foreign-types",
"libc",
@@ -5869,7 +6024,7 @@ version = "11.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d"
dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.9.0",
]
[[package]]
@@ -6185,6 +6340,31 @@ dependencies = [
"librocksdb-sys",
]
+[[package]]
+name = "rusqlite"
+version = "0.35.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a22715a5d6deef63c637207afbe68d0c72c3f8d0022d7cf9714c442d6157606b"
+dependencies = [
+ "bitflags 2.9.0",
+ "fallible-iterator",
+ "fallible-streaming-iterator",
+ "hashlink",
+ "libsqlite3-sys",
+ "smallvec",
+]
+
+[[package]]
+name = "rusqlite_migration"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6bec955369ac93728d3864d56b5d663ec7495f83648c560750a56c50fefcbf8"
+dependencies = [
+ "include_dir",
+ "log",
+ "rusqlite",
+]
+
[[package]]
name = "rust-stemmers"
version = "1.2.0"
@@ -6254,7 +6434,7 @@ version = "0.38.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949"
dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.9.0",
"errno",
"libc",
"linux-raw-sys",
@@ -6455,7 +6635,7 @@ version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c95a930e03325234c18c7071fd2b60118307e025d6fff3e12745ffbf63a3d29c"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
"cssparser",
"ego-tree",
"getopts",
@@ -6472,7 +6652,7 @@ version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "585480e3719b311b78a573db1c9d9c4c1f8010c2dee4cc59c2efe58ea4dbc3e1"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
"cssparser",
"ego-tree",
"getopts",
@@ -6537,7 +6717,7 @@ version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06"
dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.9.0",
"cssparser",
"derive_more",
"fxhash",
@@ -6671,6 +6851,31 @@ dependencies = [
"unsafe-libyaml",
]
+[[package]]
+name = "serial_test"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d"
+dependencies = [
+ "dashmap 5.5.3",
+ "futures",
+ "lazy_static",
+ "log",
+ "parking_lot 0.12.1",
+ "serial_test_derive",
+]
+
+[[package]]
+name = "serial_test_derive"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.94",
+]
+
[[package]]
name = "servo_arc"
version = "0.3.0"
@@ -6720,7 +6925,7 @@ dependencies = [
[[package]]
name = "shared-entity"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"anyhow",
"app-error",
@@ -6873,6 +7078,15 @@ version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+[[package]]
+name = "sqlite-vec"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ec77b84fb8dd5f0f8def127226db83b5d1152c5bf367f09af03998b76ba554a"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
@@ -6929,6 +7143,15 @@ version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
+[[package]]
+name = "strum"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32"
+dependencies = [
+ "strum_macros 0.27.1",
+]
+
[[package]]
name = "strum_macros"
version = "0.21.1"
@@ -6967,6 +7190,19 @@ dependencies = [
"syn 2.0.94",
]
+[[package]]
+name = "strum_macros"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 2.0.94",
+]
+
[[package]]
name = "subtle"
version = "2.5.0"
@@ -7065,7 +7301,7 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.9.0",
"core-foundation",
"system-configuration-sys 0.6.0",
]
@@ -7313,6 +7549,23 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "text-splitter"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8130aecc3b7938ce3ea387d7615eca92bd4f702a5adc0548ba930a9c039dda4"
+dependencies = [
+ "ahash 0.8.11",
+ "auto_enums",
+ "either",
+ "icu_provider",
+ "icu_segmenter",
+ "itertools 0.14.0",
+ "memchr",
+ "strum 0.27.1",
+ "thiserror 2.0.12",
+]
+
[[package]]
name = "thiserror"
version = "1.0.64"
@@ -7590,14 +7843,14 @@ dependencies = [
[[package]]
name = "toml"
-version = "0.7.8"
+version = "0.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257"
+checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
- "toml_edit 0.19.15",
+ "toml_edit 0.22.12",
]
[[package]]
@@ -7611,26 +7864,26 @@ dependencies = [
[[package]]
name = "toml_edit"
-version = "0.19.15"
+version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
+checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
dependencies = [
"indexmap 2.1.0",
- "serde",
- "serde_spanned",
"toml_datetime",
- "winnow",
+ "winnow 0.5.30",
]
[[package]]
name = "toml_edit"
-version = "0.21.1"
+version = "0.22.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
+checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef"
dependencies = [
"indexmap 2.1.0",
+ "serde",
+ "serde_spanned",
"toml_datetime",
- "winnow",
+ "winnow 0.6.26",
]
[[package]]
@@ -7748,7 +8001,7 @@ version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d637245a0d8774bd48df6482e086c59a8b5348a910c3b0579354045a9d82411"
dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
"gethostname",
"log",
"serde",
@@ -7911,6 +8164,15 @@ dependencies = [
"utf-8",
]
+[[package]]
+name = "twox-hash"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908"
+dependencies = [
+ "rand 0.8.5",
+]
+
[[package]]
name = "typenum"
version = "1.16.0"
@@ -8670,6 +8932,15 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "winnow"
+version = "0.6.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "winreg"
version = "0.50.0"
@@ -8693,7 +8964,7 @@ dependencies = [
[[package]]
name = "workspace-template"
version = "0.1.0"
-source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=6ed0f4e#6ed0f4e21c622f692f0f0b2143523e9346b395b9"
+source = "git+https://github.com/AppFlowy-IO/AppFlowy-Cloud?rev=bfe9da67254f4e17a9cb05ffb88bd4a586a71179#bfe9da67254f4e17a9cb05ffb88bd4a586a71179"
dependencies = [
"anyhow",
"async-trait",
@@ -8796,18 +9067,38 @@ dependencies = [
[[package]]
name = "zerocopy"
-version = "0.7.26"
+version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
- "zerocopy-derive",
+ "zerocopy-derive 0.7.35",
+]
+
+[[package]]
+name = "zerocopy"
+version = "0.9.0-alpha.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c9c378180b968f2e98efa0c4bba8c49fd4cd247dc82a7f01e72dc137d447d8c"
+dependencies = [
+ "zerocopy-derive 0.9.0-alpha.0",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.94",
]
[[package]]
name = "zerocopy-derive"
-version = "0.7.26"
+version = "0.9.0-alpha.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f"
+checksum = "36811c5ea90dd441f941919fc99163e702f150a8e1495d72a268106a0100df09"
dependencies = [
"proc-macro2",
"quote",
diff --git a/frontend/rust-lib/Cargo.toml b/frontend/rust-lib/Cargo.toml
index ea5db0cbc9..a98c24faf6 100644
--- a/frontend/rust-lib/Cargo.toml
+++ b/frontend/rust-lib/Cargo.toml
@@ -29,7 +29,7 @@ members = [
"flowy-search-pub",
"flowy-ai",
"flowy-ai-pub",
- "flowy-storage-pub",
+ "flowy-storage-pub", "flowy-sqlite-vec",
]
resolver = "2"
@@ -103,15 +103,16 @@ tantivy = { version = "0.24.0" }
af-plugin = { version = "0.1" }
af-local-ai = { version = "0.1" }
num_enum = "0.7.3"
+flowy-sqlite-vec = { path = "flowy-sqlite-vec" }
# Please using the following command to update the revision id
# Current directory: frontend
# Run the script.add_workspace_members:
# scripts/tool/update_client_api_rev.sh new_rev_id
# ⚠️⚠️⚠️️
-client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6ed0f4e" }
-client-api-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6ed0f4e" }
-workspace-template = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "6ed0f4e" }
+client-api = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "bfe9da67254f4e17a9cb05ffb88bd4a586a71179" }
+client-api-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "bfe9da67254f4e17a9cb05ffb88bd4a586a71179" }
+workspace-template = { git = "https://github.com/AppFlowy-IO/AppFlowy-Cloud", rev = "bfe9da67254f4e17a9cb05ffb88bd4a586a71179" }
[profile.dev]
opt-level = 0
@@ -146,14 +147,14 @@ rocksdb = { git = "https://github.com/rust-rocksdb/rust-rocksdb", rev = "1710120
# To switch to the local path, run:
# scripts/tool/update_collab_source.sh
# ⚠️⚠️⚠️️
-collab = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3" }
-collab-entity = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3" }
-collab-folder = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3" }
-collab-document = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3" }
-collab-database = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3" }
-collab-plugins = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3" }
-collab-user = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3" }
-collab-importer = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "f029a79e6112c296286cd7bb4c6dcaa4cf0d33f3" }
+collab = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe" }
+collab-entity = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe" }
+collab-folder = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe" }
+collab-document = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe" }
+collab-database = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe" }
+collab-plugins = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe" }
+collab-user = { version = "0.2", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe" }
+collab-importer = { version = "0.1", git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "b9bd12e83d3d9acacfacecf4ec12c7bae1df99fe" }
# Working directory: frontend
# To update the commit ID, run:
diff --git a/frontend/rust-lib/build-tool/flowy-ast/src/symbol.rs b/frontend/rust-lib/build-tool/flowy-ast/src/symbol.rs
index 60e994d212..341a94553f 100644
--- a/frontend/rust-lib/build-tool/flowy-ast/src/symbol.rs
+++ b/frontend/rust-lib/build-tool/flowy-ast/src/symbol.rs
@@ -53,7 +53,7 @@ impl PartialEq<Symbol> for Ident {
}
}
-impl<'a> PartialEq<Symbol> for &'a Ident {
+impl PartialEq<Symbol> for &Ident {
fn eq(&self, word: &Symbol) -> bool {
*self == word.0
}
@@ -65,7 +65,7 @@ impl PartialEq<Symbol> for Path {
}
}
-impl<'a> PartialEq<Symbol> for &'a Path {
+impl PartialEq<Symbol> for &Path {
fn eq(&self, word: &Symbol) -> bool {
self.is_ident(word.0)
}
diff --git a/frontend/rust-lib/collab-integrate/Cargo.toml b/frontend/rust-lib/collab-integrate/Cargo.toml
index 0cbdd41ccd..46c21818f1 100644
--- a/frontend/rust-lib/collab-integrate/Cargo.toml
+++ b/frontend/rust-lib/collab-integrate/Cargo.toml
@@ -22,10 +22,12 @@ tracing.workspace = true
tokio = { workspace = true, features = ["sync"] }
lib-infra = { workspace = true }
arc-swap = "1.7"
-flowy-sqlite = { workspace = true }
-diesel.workspace = true
flowy-error.workspace = true
uuid.workspace = true
+flowy-ai-pub.workspace = true
+
+[target.'cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))'.dependencies]
+twox-hash = { version = "2.1.0", features = ["xxhash64"] }
[features]
default = []
diff --git a/frontend/rust-lib/collab-integrate/src/collab_builder.rs b/frontend/rust-lib/collab-integrate/src/collab_builder.rs
index 223ebacc91..dda510a17b 100644
--- a/frontend/rust-lib/collab-integrate/src/collab_builder.rs
+++ b/frontend/rust-lib/collab-integrate/src/collab_builder.rs
@@ -33,7 +33,9 @@ use collab_plugins::local_storage::kv::KVTransactionDB;
use collab_plugins::local_storage::CollabPersistenceConfig;
use collab_user::core::{UserAwareness, UserAwarenessNotifier};
+use crate::instant_indexed_data_provider::InstantIndexedDataProvider;
use flowy_error::FlowyError;
+use lib_infra::util::get_operating_system;
use lib_infra::{if_native, if_wasm};
use tracing::{error, instrument, trace, warn};
use uuid::Uuid;
@@ -79,14 +81,17 @@ pub struct AppFlowyCollabBuilder {
#[cfg(not(target_arch = "wasm32"))]
rocksdb_backup: ArcSwapOption<Arc<dyn RocksdbBackup>>,
workspace_integrate: Arc<dyn WorkspaceCollabIntegrate>,
+ embeddings_writer: Option<Weak<InstantIndexedDataProvider>>,
}
impl AppFlowyCollabBuilder {
pub fn new(
storage_provider: impl CollabCloudPluginProvider + 'static,
workspace_integrate: impl WorkspaceCollabIntegrate + 'static,
+ embeddings_writer: Option<Weak<InstantIndexedDataProvider>>,
) -> Self {
Self {
+ embeddings_writer,
network_reachability: CollabConnectReachability::new(),
plugin_provider: ArcSwap::new(Arc::new(Arc::new(storage_provider))),
snapshot_persistence: Default::default(),
@@ -306,6 +311,19 @@ impl AppFlowyCollabBuilder {
where
T: BorrowMut<Collab> + Send + Sync + 'static,
{
+ if get_operating_system().is_desktop() {
+ let cloned_object = object.clone();
+ let weak_collab = Arc::downgrade(&collab);
+ let weak_embedding_writer = self.embeddings_writer.clone();
+ tokio::spawn(async move {
+ if let Some(embedding_writer) = weak_embedding_writer.and_then(|w| w.upgrade()) {
+ embedding_writer
+ .queue_collab_embed(cloned_object, weak_collab)
+ .await;
+ }
+ });
+ }
+
let mut write_collab = collab.try_write()?;
let has_cloud_plugin = write_collab.borrow().has_cloud_plugin();
if has_cloud_plugin {
diff --git a/frontend/rust-lib/collab-integrate/src/instant_indexed_data_provider.rs b/frontend/rust-lib/collab-integrate/src/instant_indexed_data_provider.rs
new file mode 100644
index 0000000000..9147058fa4
--- /dev/null
+++ b/frontend/rust-lib/collab-integrate/src/instant_indexed_data_provider.rs
@@ -0,0 +1,226 @@
+use collab::lock::RwLock;
+use collab::preclude::{Collab, Transact};
+use collab_document::document::DocumentBody;
+use collab_entity::{CollabObject, CollabType};
+use flowy_ai_pub::entities::UnindexedData;
+use flowy_error::{FlowyError, FlowyResult};
+use lib_infra::async_trait::async_trait;
+use lib_infra::util::get_operating_system;
+use std::borrow::BorrowMut;
+use std::collections::HashMap;
+use std::sync::{Arc, Weak};
+use std::time::Duration;
+use tokio::runtime::Runtime;
+use tokio::time::interval;
+use tracing::{error, info, trace};
+use uuid::Uuid;
+
+pub struct WriteObject {
+ pub collab_object: CollabObject,
+ pub collab: Weak<dyn CollabIndexedData>,
+}
+
+pub struct InstantIndexedDataProvider {
+ collab_by_object: Arc<RwLock<HashMap<String, WriteObject>>>,
+ consumers: Arc<RwLock<Vec<Box<dyn InstantIndexedDataConsumer>>>>,
+}
+
+impl Default for InstantIndexedDataProvider {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl InstantIndexedDataProvider {
+ pub fn new() -> InstantIndexedDataProvider {
+ let collab_by_object = Arc::new(RwLock::new(HashMap::<String, WriteObject>::new()));
+ let consumers = Arc::new(RwLock::new(
+ Vec::<Box<dyn InstantIndexedDataConsumer>>::new(),
+ ));
+
+ InstantIndexedDataProvider {
+ collab_by_object,
+ consumers,
+ }
+ }
+
+ pub async fn num_consumers(&self) -> usize {
+ let consumers = self.consumers.read().await;
+ consumers.len()
+ }
+
+ pub async fn clear_consumers(&self) {
+ let mut consumers = self.consumers.write().await;
+ consumers.clear();
+ info!("[Indexing] Cleared all instant index consumers");
+ }
+
+ pub async fn register_consumer(&self, consumer: Box<dyn InstantIndexedDataConsumer>) {
+ info!(
+ "[Indexing] Registering instant index consumer: {}",
+ consumer.consumer_id()
+ );
+ let mut guard = self.consumers.write().await;
+ guard.push(consumer);
+ }
+
+ pub async fn spawn_instant_indexed_provider(&self, runtime: &Runtime) -> FlowyResult<()> {
+ if !get_operating_system().is_desktop() {
+ return Ok(());
+ }
+
+ let weak_collab_by_object = Arc::downgrade(&self.collab_by_object);
+ let consumers_weak = Arc::downgrade(&self.consumers);
+ let interval_dur = Duration::from_secs(30);
+
+ runtime.spawn(async move {
+ let mut ticker = interval(interval_dur);
+ ticker.tick().await;
+
+ loop {
+ ticker.tick().await;
+
+ // Upgrade our state holders
+ let collab_by_object = match weak_collab_by_object.upgrade() {
+ Some(m) => m,
+ None => break, // provider dropped
+ };
+ let consumers = match consumers_weak.upgrade() {
+ Some(c) => c,
+ None => break,
+ };
+
+ // Snapshot keys and consumers under read locks
+ let (object_ids, mut to_remove) = {
+ let guard = collab_by_object.read().await;
+ let keys: Vec<_> = guard.keys().cloned().collect();
+ (keys, Vec::new())
+ };
+ let guard = collab_by_object.read().await;
+ for id in object_ids {
+ // Check if the collab is still alive
+ match guard.get(&id) {
+ Some(wo) => {
+ if let Some(collab_rc) = wo.collab.upgrade() {
+ if let Some(data) = collab_rc
+ .get_unindexed_data(&wo.collab_object.collab_type)
+ .await
+ {
+ // Snapshot consumers
+ let consumers_guard = consumers.read().await;
+ for consumer in consumers_guard.iter() {
+ match consumer
+ .consume_collab(&wo.collab_object, data.clone())
+ .await
+ {
+ Ok(is_indexed) => {
+ if is_indexed {
+ trace!("[Indexing] {} consumed {}", consumer.consumer_id(), id);
+ }
+ },
+ Err(err) => {
+ error!(
+ "Consumer {} failed on {}: {}",
+ consumer.consumer_id(),
+ id,
+ err
+ );
+ },
+ }
+ }
+ }
+ } else {
+ // Mark for removal if collab was dropped
+ to_remove.push(id);
+ }
+ },
+ None => continue,
+ }
+ }
+
+ if !to_remove.is_empty() {
+ let mut guard = collab_by_object.write().await;
+ guard.retain(|k, _| !to_remove.contains(k));
+ trace!("[Indexing] Removed {} stale entries", to_remove.len());
+ }
+ }
+
+ info!("[Indexing] Instant indexed data provider stopped");
+ });
+
+ Ok(())
+ }
+
+ pub fn support_collab_type(&self, t: &CollabType) -> bool {
+ matches!(t, CollabType::Document)
+ }
+
+ pub async fn queue_collab_embed(
+ &self,
+ collab_object: CollabObject,
+ collab: Weak<dyn CollabIndexedData>,
+ ) {
+ if !get_operating_system().is_desktop() {
+ return;
+ }
+
+ if !self.support_collab_type(&collab_object.collab_type) {
+ return;
+ }
+
+ let mut map = self.collab_by_object.write().await;
+ map.insert(
+ collab_object.object_id.clone(),
+ WriteObject {
+ collab_object,
+ collab,
+ },
+ );
+ }
+}
+
+fn index_data_for_collab(collab: &Collab, collab_type: &CollabType) -> Option<UnindexedData> {
+ match collab_type {
+ CollabType::Document => {
+ let txn = collab.doc().try_transact().ok()?;
+ let doc = DocumentBody::from_collab(collab)?;
+ let paras = doc.paragraphs(txn);
+ Some(UnindexedData::Paragraphs(paras))
+ },
+ _ => None,
+ }
+}
+
+#[async_trait]
+pub trait CollabIndexedData: Send + Sync + 'static {
+ async fn get_unindexed_data(&self, collab_type: &CollabType) -> Option<UnindexedData>;
+}
+
+#[async_trait]
+impl<T> CollabIndexedData for RwLock<T>
+where
+ T: BorrowMut<Collab> + Send + Sync + 'static,
+{
+ async fn get_unindexed_data(&self, collab_type: &CollabType) -> Option<UnindexedData> {
+ let collab = self.try_read().ok()?;
+ index_data_for_collab(collab.borrow(), collab_type)
+ }
+}
+
+/// writer interface
+#[async_trait]
+pub trait InstantIndexedDataConsumer: Send + Sync + 'static {
+ fn consumer_id(&self) -> String;
+
+ async fn consume_collab(
+ &self,
+ collab_object: &CollabObject,
+ data: UnindexedData,
+ ) -> Result<bool, FlowyError>;
+
+ async fn did_delete_collab(
+ &self,
+ workspace_id: &Uuid,
+ object_id: &Uuid,
+ ) -> Result<(), FlowyError>;
+}
diff --git a/frontend/rust-lib/collab-integrate/src/lib.rs b/frontend/rust-lib/collab-integrate/src/lib.rs
index afa6f0c2a8..e4bde9c3e3 100644
--- a/frontend/rust-lib/collab-integrate/src/lib.rs
+++ b/frontend/rust-lib/collab-integrate/src/lib.rs
@@ -4,7 +4,7 @@ pub use collab_plugins::CollabKVDB;
pub mod collab_builder;
pub mod config;
-pub mod persistence;
+pub mod instant_indexed_data_provider;
mod plugin_provider;
pub use collab_plugins::local_storage::kv::doc::CollabKVAction;
diff --git a/frontend/rust-lib/collab-integrate/src/persistence/mod.rs b/frontend/rust-lib/collab-integrate/src/persistence/mod.rs
deleted file mode 100644
index dc0eb77c28..0000000000
--- a/frontend/rust-lib/collab-integrate/src/persistence/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod collab_metadata_sql;
diff --git a/frontend/rust-lib/dart-ffi/src/lib.rs b/frontend/rust-lib/dart-ffi/src/lib.rs
index 6c3d08ae01..ac2767b6fc 100644
--- a/frontend/rust-lib/dart-ffi/src/lib.rs
+++ b/frontend/rust-lib/dart-ffi/src/lib.rs
@@ -84,13 +84,25 @@ impl DartAppFlowyCore {
ret: Option<mpsc::Sender<AFPluginEventResponse>>,
) {
if let Ok(sender_guard) = self.sender.read() {
- if let Err(e) = sender_guard.as_ref().unwrap().send(Task {
- dispatcher: self.dispatcher().unwrap(),
- request,
- port,
- ret,
- }) {
- error!("Failed to send task: {}", e);
+ let dispatcher = match self.dispatcher() {
+ Some(dispatcher) => dispatcher,
+ None => {
+ error!("Failed to get dispatcher: dispatcher is None");
+ return;
+ },
+ };
+
+ if let Some(sender) = sender_guard.as_ref() {
+ if let Err(e) = sender.send(Task {
+ dispatcher,
+ request,
+ port,
+ ret,
+ }) {
+ error!("Failed to send task: {}", e);
+ }
+ } else {
+ error!("Failed to send task: sender is None");
}
} else {
warn!("Failed to acquire read lock for sender");
diff --git a/frontend/rust-lib/event-integration-test/Cargo.toml b/frontend/rust-lib/event-integration-test/Cargo.toml
index 6b2d5af7ba..56ed24d341 100644
--- a/frontend/rust-lib/event-integration-test/Cargo.toml
+++ b/frontend/rust-lib/event-integration-test/Cargo.toml
@@ -48,9 +48,8 @@ zip.workspace = true
walkdir = "2.5.0"
futures = "0.3.31"
flowy-ai-pub = { workspace = true }
+serial_test = "2.0.0"
[features]
-default = ["supabase_cloud_test"]
dart = ["flowy-core/dart"]
-supabase_cloud_test = []
-single_thread = []
\ No newline at end of file
+single_thread = []
diff --git a/frontend/rust-lib/event-integration-test/src/folder_event.rs b/frontend/rust-lib/event-integration-test/src/folder_event.rs
index 26515ab5af..4ed530c181 100644
--- a/frontend/rust-lib/event-integration-test/src/folder_event.rs
+++ b/frontend/rust-lib/event-integration-test/src/folder_event.rs
@@ -112,6 +112,15 @@ impl EventIntegrationTest {
.parse::<WorkspacePB>()
}
+ pub async fn get_workspace_id(&self) -> Uuid {
+ let a = EventBuilder::new(self.clone())
+ .event(FolderEvent::ReadCurrentWorkspace)
+ .async_send()
+ .await
+ .parse::<WorkspacePB>();
+ Uuid::from_str(&a.id).unwrap()
+ }
+
pub async fn get_user_workspace(&self, workspace_id: &str) -> UserWorkspacePB {
let payload = UserWorkspaceIdPB {
workspace_id: workspace_id.to_string(),
@@ -124,7 +133,7 @@ impl EventIntegrationTest {
.parse::<UserWorkspacePB>()
}
- pub fn get_folder_search_handler(&self) -> &Arc<dyn SearchHandler> {
+ pub fn get_folder_search_handler(&self) -> Arc<dyn SearchHandler> {
self
.appflowy_core
.search_manager
diff --git a/frontend/rust-lib/event-integration-test/src/user_event.rs b/frontend/rust-lib/event-integration-test/src/user_event.rs
index 6e1bfcd80e..8486555840 100644
--- a/frontend/rust-lib/event-integration-test/src/user_event.rs
+++ b/frontend/rust-lib/event-integration-test/src/user_event.rs
@@ -10,6 +10,8 @@ use tokio::sync::broadcast::{channel, Sender};
use tracing::error;
use uuid::Uuid;
+use crate::event_builder::EventBuilder;
+use crate::EventIntegrationTest;
use flowy_folder::event_map::FolderEvent;
use flowy_notification::entities::SubscribeObject;
use flowy_notification::NotificationSender;
@@ -27,9 +29,6 @@ use flowy_user::event_map::UserEvent;
use flowy_user_pub::entities::WorkspaceType;
use lib_dispatch::prelude::{AFPluginDispatcher, AFPluginRequest, ToBytes};
-use crate::event_builder::EventBuilder;
-use crate::EventIntegrationTest;
-
impl EventIntegrationTest {
pub async fn enable_encryption(&self) -> String {
let config = EventBuilder::new(self.clone())
diff --git a/frontend/rust-lib/event-integration-test/tests/asset/090_anon_search.zip b/frontend/rust-lib/event-integration-test/tests/asset/090_anon_search.zip
new file mode 100644
index 0000000000..33c53c1cc1
Binary files /dev/null and b/frontend/rust-lib/event-integration-test/tests/asset/090_anon_search.zip differ
diff --git a/frontend/rust-lib/event-integration-test/tests/document/mod.rs b/frontend/rust-lib/event-integration-test/tests/document/mod.rs
index e11cb782f4..ff3f62e793 100644
--- a/frontend/rust-lib/event-integration-test/tests/document/mod.rs
+++ b/frontend/rust-lib/event-integration-test/tests/document/mod.rs
@@ -1,8 +1,6 @@
mod local_test;
mod af_cloud_test;
-// #[cfg(feature = "supabase_cloud_test")]
-// mod supabase_test;
use rand::{distributions::Alphanumeric, thread_rng, Rng};
diff --git a/frontend/rust-lib/event-integration-test/tests/folder/mod.rs b/frontend/rust-lib/event-integration-test/tests/folder/mod.rs
index c5566e1b80..a8bc6de63f 100644
--- a/frontend/rust-lib/event-integration-test/tests/folder/mod.rs
+++ b/frontend/rust-lib/event-integration-test/tests/folder/mod.rs
@@ -1,3 +1 @@
mod local_test;
-// #[cfg(feature = "supabase_cloud_test")]
-// mod supabase_test;
diff --git a/frontend/rust-lib/event-integration-test/tests/main.rs b/frontend/rust-lib/event-integration-test/tests/main.rs
index cf4c1591ac..18d9eb99f1 100644
--- a/frontend/rust-lib/event-integration-test/tests/main.rs
+++ b/frontend/rust-lib/event-integration-test/tests/main.rs
@@ -2,8 +2,7 @@ mod database;
mod document;
mod folder;
-// TODO(Mathias): Enable tests for search
-// mod search;
+mod search;
mod sql_test;
mod user;
diff --git a/frontend/rust-lib/event-integration-test/tests/search/document_title_content_search.rs b/frontend/rust-lib/event-integration-test/tests/search/document_title_content_search.rs
new file mode 100644
index 0000000000..dd07be9691
--- /dev/null
+++ b/frontend/rust-lib/event-integration-test/tests/search/document_title_content_search.rs
@@ -0,0 +1,207 @@
+use crate::util::unzip;
+use event_integration_test::EventIntegrationTest;
+use flowy_core::DEFAULT_NAME;
+use flowy_search::entities::SearchResponsePB;
+use flowy_search::services::manager::SearchType;
+use flowy_user::errors::FlowyResult;
+use flowy_user_pub::entities::WorkspaceType;
+use futures::StreamExt;
+use std::time::{Duration, Instant};
+use uuid::Uuid;
+
+// Helper function to wait for search indexing to complete
+async fn wait_for_indexing(test: &EventIntegrationTest) {
+ let mut rx = test
+ .user_manager
+ .app_life_cycle
+ .read()
+ .await
+ .subscribe_full_indexed_finish()
+ .unwrap();
+ let _ = rx.changed().await;
+}
+
+// Helper function to perform search and collect results
+async fn perform_search(
+ test: &EventIntegrationTest,
+ query: &str,
+ workspace_id: &Uuid,
+) -> Vec<FlowyResult<SearchResponsePB>> {
+ let search_handler = test
+ .search_manager
+ .get_handler(SearchType::DocumentLocal)
+ .unwrap();
+
+ let stream = search_handler
+ .perform_search(query.to_string(), workspace_id)
+ .await;
+
+ stream.collect().await
+}
+
+// Helper function to wait for document to be indexed with a specified query
+async fn wait_for_document_indexing(
+ test: &EventIntegrationTest,
+ query: &str,
+ workspace_id: &Uuid,
+ document_name: &str,
+ timeout_secs: u64,
+) -> Vec<FlowyResult<SearchResponsePB>> {
+ let start_time = Instant::now();
+ let timeout = Duration::from_secs(timeout_secs);
+ let mut result = Vec::new();
+
+ while start_time.elapsed() < timeout {
+ result = perform_search(test, query, workspace_id).await;
+
+ if let Some(Ok(search_result)) = result.first() {
+ if let Some(local) = &search_result.local_search_result {
+ if local
+ .items
+ .iter()
+ .any(|item| item.display_name.contains(document_name))
+ {
+ break;
+ }
+ }
+ }
+
+ tokio::time::sleep(Duration::from_secs(2)).await;
+ }
+
+ result
+}
+
+#[tokio::test]
+async fn anon_user_multiple_workspace_search_test() {
+ // SETUP: Initialize test environment with test data
+ let user_db_path = unzip("./tests/asset", "090_anon_search").unwrap();
+ let test =
+ EventIntegrationTest::new_with_user_data_path(user_db_path, DEFAULT_NAME.to_string()).await;
+ let first_workspace_id = test.get_workspace_id().await;
+
+ // Wait for initial indexing to complete
+ wait_for_indexing(&test).await;
+ // TEST CASE 1: Search by page title
+ let result = perform_search(&test, "japan", &first_workspace_id).await;
+ let local = result[0]
+ .as_ref()
+ .unwrap()
+ .local_search_result
+ .as_ref()
+ .expect("expected a local_search_result");
+
+ assert_eq!(
+ local.items.len(),
+ 2,
+ "Should find 2 pages with 'japan' in the title"
+ );
+ assert_eq!(
+ local.items[0].display_name, "Japan Skiing",
+ "First result should be 'Japan Skiing'"
+ );
+ assert_eq!(
+ local.items[1].display_name, "Japan Food",
+ "Second result should be 'Japan Food'"
+ );
+
+ // TEST CASE 2: Search by page content
+ let result = perform_search(&test, "Niseko", &first_workspace_id).await;
+ let local = result[0]
+ .as_ref()
+ .unwrap()
+ .local_search_result
+ .as_ref()
+ .expect("expected a local_search_result");
+
+ assert_eq!(
+ local.items.len(),
+ 1,
+ "Should find 1 page with 'Niseko' in the content"
+ );
+ assert_eq!(
+ local.items[0].display_name, "Japan Skiing",
+ "The page should be 'Japan Skiing'"
+ );
+
+ // TEST CASE 3: Create a new document and verify it becomes searchable
+ // Create and add content to new document
+ let document_title = "My dog";
+ let view = test
+ .create_and_open_document(
+ &first_workspace_id.to_string(),
+ document_title.to_string(),
+ vec![],
+ )
+ .await;
+ test
+ .insert_document_text(
+ &view.id,
+ "I have maltese dog, he love eating food all the time",
+ 0,
+ )
+ .await;
+
+ // Wait for document to be indexed and searchable
+ let result = wait_for_document_indexing(
+ &test,
+ "maltese dog",
+ &first_workspace_id,
+ document_title,
+ 30,
+ )
+ .await;
+
+ let local = result[0]
+ .as_ref()
+ .unwrap()
+ .local_search_result
+ .as_ref()
+ .expect("expected a local_search_result");
+
+ assert!(
+ local
+ .items
+ .iter()
+ .any(|item| item.display_name.contains(document_title)),
+ "New document should be found when searching for its content"
+ );
+
+ // TEST CASE 4: Create and search in a second workspace
+ // Create and open a new workspace
+ let second_workspace_id = Uuid::parse_str(
+ &test
+ .create_workspace("my second workspace", WorkspaceType::Local)
+ .await
+ .workspace_id,
+ )
+ .unwrap();
+
+ test
+ .open_workspace(
+ &second_workspace_id.to_string(),
+ WorkspaceType::Local.into(),
+ )
+ .await;
+
+ // Wait for indexing in the new workspace
+ wait_for_indexing(&test).await;
+
+ // Search in second workspace
+ let result = perform_search(&test, "japan", &second_workspace_id).await;
+ assert!(
+ result[0].as_ref().unwrap().local_search_result.is_none(),
+ "Empty workspace should not have results for 'japan'"
+ );
+
+ // TEST CASE 5: Return to first workspace and verify search still works
+ test
+ .open_workspace(&first_workspace_id.to_string(), WorkspaceType::Local.into())
+ .await;
+ wait_for_indexing(&test).await;
+ let result = perform_search(&test, "japan", &first_workspace_id).await;
+ assert!(
+ !result.is_empty(),
+ "First workspace should still have search results after switching workspaces"
+ );
+}
diff --git a/frontend/rust-lib/event-integration-test/tests/search/local_test/folder_search_test.rs b/frontend/rust-lib/event-integration-test/tests/search/local_test/folder_search_test.rs
deleted file mode 100644
index 995854699a..0000000000
--- a/frontend/rust-lib/event-integration-test/tests/search/local_test/folder_search_test.rs
+++ /dev/null
@@ -1,215 +0,0 @@
-use crate::util::{unzip_test_asset, zip};
-use collab_folder::View;
-use event_integration_test::EventIntegrationTest;
-use flowy_core::DEFAULT_NAME;
-use flowy_folder::entities::UpdateViewPayloadPB;
-use flowy_folder_pub::folder_builder::{FlattedViews, NestedViewBuilder};
-use std::time::Duration;
-use tokio::time::sleep;
-
-#[tokio::test]
-async fn test_folder_index_all_startup() {
- let folder_name = "folder_1000_view";
- // comment out the following line to create a test asset if you modify the test data
- // don't forget to delete unnecessary test assets
- // create_folder_test_data(folder_name).await;
-
- let (cleaner, user_db_path) = unzip_test_asset(folder_name).unwrap();
- let test =
- EventIntegrationTest::new_with_user_data_path(user_db_path.clone(), DEFAULT_NAME.to_string())
- .await;
-
- let first_level_views = test.get_all_workspace_views().await;
- assert_eq!(first_level_views.len(), 3);
- assert_eq!(first_level_views[1].name, "1");
- assert_eq!(first_level_views[2].name, "2");
-
- let view_1 = test.get_view(&first_level_views[1].id).await;
- assert_eq!(view_1.child_views.len(), 500);
-
- let folder_data = test.get_folder_data();
- // Get started + 1002 Views
- assert_eq!(folder_data.views.len(), 1003);
-
- // Wait for the index to be created/updated
- sleep(Duration::from_secs(1)).await;
-
- let folder_search_manager = test.get_folder_search_handler();
- let num_docs = folder_search_manager.index_count();
- assert_eq!(num_docs, 1004);
-
- drop(cleaner);
-}
-
-#[tokio::test]
-async fn test_folder_index_create_20_views() {
- let test = EventIntegrationTest::new_anon().await;
- let folder_search_manager = test.get_folder_search_handler();
-
- // Wait for the index to be created/updated
- sleep(Duration::from_secs(1)).await;
- let workspace_id = test.get_current_workspace().await.id;
-
- for i in 0..20 {
- let view = test.create_view(&workspace_id, format!("View {}", i)).await;
- sleep(Duration::from_millis(500)).await;
- assert_eq!(view.name, format!("View {}", i));
- }
-
- // Wait for the index update to finish
- sleep(Duration::from_secs(2)).await;
-
- let num_docs = folder_search_manager.index_count();
- // Workspace + Get started + 20 Views
- assert_eq!(num_docs, 22);
-}
-
-#[tokio::test]
-async fn test_folder_index_create_view() {
- let test = EventIntegrationTest::new_anon().await;
-
- let folder_search_manager = test.get_folder_search_handler();
-
- // Wait for the index to be created/updated
- sleep(Duration::from_secs(1)).await;
-
- let workspace_id = test.get_current_workspace().await.id;
- let view = test.create_view(&workspace_id, "Flowers".to_owned()).await;
-
- // Wait for the index to be updated
- sleep(Duration::from_millis(500)).await;
-
- let results = folder_search_manager.perform_search(view.name.clone(), None);
- if let Err(e) = results {
- panic!("Error performing search: {:?}", e);
- }
-
- let results = results.unwrap();
- assert_eq!(results.len(), 1);
- assert_eq!(results[0].data, view.name);
-}
-
-#[tokio::test]
-async fn test_folder_index_rename_view() {
- let test = EventIntegrationTest::new_anon().await;
- let folder_search_manager = test.get_folder_search_handler();
-
- // Wait for the index to be created/updated
- sleep(Duration::from_secs(1)).await;
-
- let workspace_id = test.get_current_workspace().await.id;
- let view = test.create_view(&workspace_id, "Flowers".to_owned()).await;
-
- // Wait for the index to be updated
- sleep(Duration::from_millis(500)).await;
-
- let new_view_name = "Bouquets".to_string();
- let update_payload = UpdateViewPayloadPB {
- view_id: view.id,
- name: Some(new_view_name.clone()),
- ..Default::default()
- };
- test.update_view(update_payload).await;
-
- // Wait for the index to be updated
- sleep(Duration::from_millis(500)).await;
-
- let first = folder_search_manager.perform_search(view.name, None);
- if let Err(e) = first {
- panic!("Error performing search: {:?}", e);
- }
-
- let second = folder_search_manager.perform_search(new_view_name.clone(), None);
- if let Err(e) = second {
- panic!("Error performing search: {:?}", e);
- }
-
- let first = first.unwrap();
- assert_eq!(first.len(), 0);
-
- let second = second.unwrap();
- assert_eq!(second.len(), 1);
- assert_eq!(second[0].data, new_view_name);
-}
-
-/// Using this method to create a folder test asset. Only use when you want to create a new asset.
-/// The file will be created at tests/asset/{file_name}.zip and it will be committed to the repo.
-///
-#[allow(dead_code)]
-async fn create_folder_test_data(file_name: &str) {
- let test = EventIntegrationTest::new_with_name(DEFAULT_NAME).await;
- test.sign_up_as_anon().await;
-
- let uid = test.get_user_profile().await.unwrap().id;
- let workspace_id = test.get_current_workspace().await.id;
- let views = create_1002_views(uid, workspace_id.clone()).await;
- test.create_views(views).await;
-
- let first_level_views = test.get_all_workspace_views().await;
- assert_eq!(first_level_views.len(), 3);
- assert_eq!(first_level_views[1].name, "1");
- assert_eq!(first_level_views[2].name, "2");
-
- let view_1 = test.get_view(&first_level_views[1].id).await;
- assert_eq!(view_1.child_views.len(), 500);
-
- let folder_data = test.get_folder_data();
- // Get started + 1002 Views
- assert_eq!(folder_data.views.len(), 1003);
-
- let data_path = test.config.application_path.clone();
- zip(
- data_path.into(),
- format!("tests/asset/{}.zip", file_name).into(),
- )
- .unwrap();
- sleep(Duration::from_secs(2)).await;
-}
-
-/// Create view without create the view's content(document/database).
-/// workspace
-/// - get_started
-/// - view_1
-/// - view_1_1
-/// - view_1_2
-/// - view_2
-/// - view_2_1
-/// - view_2_2
-async fn create_1002_views(uid: i64, workspace_id: String) -> Vec<View> {
- let mut builder = NestedViewBuilder::new(workspace_id.clone(), uid);
- builder
- .with_view_builder(|view_builder| async {
- let mut builder = view_builder.with_name("1");
- for i in 0..500 {
- builder = builder
- .with_child_view_builder(|child_view_builder| async {
- child_view_builder.with_name(format!("1_{}", i)).build()
- })
- .await;
- }
- builder.build()
- })
- .await;
- builder
- .with_view_builder(|view_builder| async {
- let mut builder = view_builder.with_name("2");
- for i in 0..500 {
- builder = builder
- .with_child_view_builder(|child_view_builder| async {
- child_view_builder.with_name(format!("2_{}", i)).build()
- })
- .await;
- }
- builder.build()
- })
- .await;
- // The output views should be:
- // view_1
- // view_1_1
- // view_1_x
- // view_2
- // view_2_1
- // view_2_x
- let views = builder.build();
- FlattedViews::flatten_views(views)
-}
diff --git a/frontend/rust-lib/event-integration-test/tests/search/local_test/mod.rs b/frontend/rust-lib/event-integration-test/tests/search/local_test/mod.rs
deleted file mode 100644
index be0f274302..0000000000
--- a/frontend/rust-lib/event-integration-test/tests/search/local_test/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-mod folder_search_test;
diff --git a/frontend/rust-lib/event-integration-test/tests/search/mod.rs b/frontend/rust-lib/event-integration-test/tests/search/mod.rs
index a8bc6de63f..519bf283e1 100644
--- a/frontend/rust-lib/event-integration-test/tests/search/mod.rs
+++ b/frontend/rust-lib/event-integration-test/tests/search/mod.rs
@@ -1 +1 @@
-mod local_test;
+mod document_title_content_search;
diff --git a/frontend/rust-lib/flowy-ai-pub/Cargo.toml b/frontend/rust-lib/flowy-ai-pub/Cargo.toml
index 93ea79bcab..f463e76fd0 100644
--- a/frontend/rust-lib/flowy-ai-pub/Cargo.toml
+++ b/frontend/rust-lib/flowy-ai-pub/Cargo.toml
@@ -13,4 +13,7 @@ futures.workspace = true
serde_json.workspace = true
serde.workspace = true
uuid.workspace = true
-flowy-sqlite = { workspace = true }
\ No newline at end of file
+flowy-sqlite = { workspace = true }
+chrono.workspace = true
+twox-hash = { version = "2.1.0", features = ["xxhash64"] }
+
diff --git a/frontend/rust-lib/flowy-ai-pub/src/entities.rs b/frontend/rust-lib/flowy-ai-pub/src/entities.rs
new file mode 100644
index 0000000000..9224379584
--- /dev/null
+++ b/frontend/rust-lib/flowy-ai-pub/src/entities.rs
@@ -0,0 +1,81 @@
+use crate::cloud::workspace_dto::ViewIcon;
+use crate::cloud::CollabType;
+use twox_hash::xxhash64::Hasher;
+use uuid::Uuid;
+
+pub struct EmbeddingRecord {
+ pub workspace_id: Uuid,
+ pub object_id: Uuid,
+ pub chunks: Vec<EmbeddedChunk>,
+}
+
+#[derive(Debug, Clone)]
+pub struct UnindexedCollabMetadata {
+ pub name: String,
+ pub icon: Option<ViewIcon>,
+}
+
+#[derive(Debug, Clone)]
+pub struct UnindexedCollab {
+ pub workspace_id: Uuid,
+ pub object_id: Uuid,
+ pub collab_type: CollabType,
+ pub data: UnindexedData,
+ pub metadata: UnindexedCollabMetadata,
+}
+
+#[derive(Debug, Clone)]
+pub enum UnindexedData {
+ Text(String),
+ Paragraphs(Vec<String>),
+}
+
+impl UnindexedData {
+ pub fn is_empty(&self) -> bool {
+ match self {
+ UnindexedData::Text(text) => text.is_empty(),
+ UnindexedData::Paragraphs(paragraphs) => paragraphs.is_empty(),
+ }
+ }
+
+ pub fn into_string(&self) -> String {
+ match self {
+ UnindexedData::Text(text) => text.clone(),
+ UnindexedData::Paragraphs(paragraphs) => paragraphs.join("\n"),
+ }
+ }
+
+ pub fn content_hash(&self) -> String {
+ match self {
+ UnindexedData::Text(text) => {
+ let h = Hasher::oneshot(0, text.as_bytes());
+ format!("{:016x}", h)
+ },
+ UnindexedData::Paragraphs(paragraphs) => {
+ let combined = paragraphs.join("");
+ let h = Hasher::oneshot(0, combined.as_bytes());
+ format!("{:016x}", h)
+ },
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct EmbeddedChunk {
+ pub fragment_id: String,
+ pub object_id: String,
+ pub content_type: i32,
+ pub content: Option<String>,
+ pub metadata: Option<String>,
+ pub fragment_index: i32,
+ pub embedder_type: i32,
+ pub embeddings: Option<Vec<f32>>,
+}
+
+#[derive(Debug, Clone)]
+pub struct SearchResult {
+ pub oid: Uuid,
+ pub content: String,
+ pub metadata: Option<serde_json::Value>,
+ pub score: f32,
+}
diff --git a/frontend/rust-lib/flowy-ai-pub/src/lib.rs b/frontend/rust-lib/flowy-ai-pub/src/lib.rs
index df7dc957e2..c52d9806bd 100644
--- a/frontend/rust-lib/flowy-ai-pub/src/lib.rs
+++ b/frontend/rust-lib/flowy-ai-pub/src/lib.rs
@@ -1,3 +1,4 @@
pub mod cloud;
+pub mod entities;
pub mod persistence;
pub mod user_service;
diff --git a/frontend/rust-lib/collab-integrate/src/persistence/collab_metadata_sql.rs b/frontend/rust-lib/flowy-ai-pub/src/persistence/collab_metadata_sql.rs
similarity index 100%
rename from frontend/rust-lib/collab-integrate/src/persistence/collab_metadata_sql.rs
rename to frontend/rust-lib/flowy-ai-pub/src/persistence/collab_metadata_sql.rs
diff --git a/frontend/rust-lib/flowy-ai-pub/src/persistence/collab_sql.rs b/frontend/rust-lib/flowy-ai-pub/src/persistence/collab_sql.rs
new file mode 100644
index 0000000000..328400aa68
--- /dev/null
+++ b/frontend/rust-lib/flowy-ai-pub/src/persistence/collab_sql.rs
@@ -0,0 +1,97 @@
+use diesel::sqlite::SqliteConnection;
+use flowy_error::FlowyResult;
+use flowy_sqlite::upsert::excluded;
+use flowy_sqlite::{
+ diesel,
+ query_dsl::*,
+ schema::{index_collab_record_table, index_collab_record_table::dsl},
+ AsChangeset, ExpressionMethods, Identifiable, Insertable, Queryable,
+};
+
+#[derive(Clone, Debug, Queryable, Insertable, AsChangeset, Identifiable)]
+#[diesel(table_name = index_collab_record_table)]
+#[diesel(primary_key(oid))]
+pub struct IndexCollabRecordTable {
+ pub oid: String,
+ pub workspace_id: String,
+ pub content_hash: String,
+}
+
+impl IndexCollabRecordTable {
+ pub fn new(workspace_id: String, oid: String, content_hash: String) -> Self {
+ Self {
+ workspace_id,
+ oid,
+ content_hash,
+ }
+ }
+}
+
+pub fn batch_upsert_index_collab(
+ conn: &mut SqliteConnection,
+ rows: Vec<IndexCollabRecordTable>,
+) -> FlowyResult<()> {
+ if rows.is_empty() {
+ return Ok(());
+ }
+
+ conn.immediate_transaction::<_, diesel::result::Error, _>(|conn| {
+ for row in rows {
+ diesel::insert_into(index_collab_record_table::table)
+ .values(&row)
+ .on_conflict(index_collab_record_table::oid)
+ .do_update()
+ .set(
+ index_collab_record_table::content_hash
+ .eq(excluded(index_collab_record_table::content_hash)),
+ )
+ .execute(conn)?;
+ }
+ Ok(())
+ })?;
+
+ Ok(())
+}
+
+pub fn upsert_index_collab(
+ conn: &mut SqliteConnection,
+ row: IndexCollabRecordTable,
+) -> FlowyResult<()> {
+ diesel::insert_into(index_collab_record_table::table)
+ .values(row)
+ .on_conflict(index_collab_record_table::oid)
+ .do_update()
+ .set((
+ index_collab_record_table::content_hash.eq(excluded(index_collab_record_table::content_hash)),
+ ))
+ .execute(conn)?;
+
+ Ok(())
+}
+
+pub fn select_indexed_collab_ids(
+ conn: &mut SqliteConnection,
+ workspace_id: String,
+) -> FlowyResult<Vec<String>> {
+ let result = index_collab_record_table::table
+ .filter(index_collab_record_table::workspace_id.eq(workspace_id))
+ .select(dsl::oid)
+ .load::<String>(conn)?;
+
+ Ok(result)
+}
+
+pub fn select_indexed_collab(
+ conn: &mut SqliteConnection,
+ workspace_id: String,
+ limit: i64,
+ offset: i64,
+) -> FlowyResult<Vec<IndexCollabRecordTable>> {
+ let result = index_collab_record_table::table
+ .filter(index_collab_record_table::workspace_id.eq(workspace_id))
+ .limit(limit)
+ .offset(offset)
+ .load::<IndexCollabRecordTable>(conn)?;
+
+ Ok(result)
+}
diff --git a/frontend/rust-lib/flowy-ai-pub/src/persistence/mod.rs b/frontend/rust-lib/flowy-ai-pub/src/persistence/mod.rs
index 7ae97148ce..a4f26cdbfb 100644
--- a/frontend/rust-lib/flowy-ai-pub/src/persistence/mod.rs
+++ b/frontend/rust-lib/flowy-ai-pub/src/persistence/mod.rs
@@ -1,7 +1,11 @@
mod chat_message_sql;
mod chat_sql;
+mod collab_metadata_sql;
+mod collab_sql;
mod local_model_sql;
pub use chat_message_sql::*;
pub use chat_sql::*;
+pub use collab_metadata_sql::*;
+pub use collab_sql::*;
pub use local_model_sql::*;
diff --git a/frontend/rust-lib/flowy-ai/Cargo.toml b/frontend/rust-lib/flowy-ai/Cargo.toml
index f948e6e00c..fd11419f6e 100644
--- a/frontend/rust-lib/flowy-ai/Cargo.toml
+++ b/frontend/rust-lib/flowy-ai/Cargo.toml
@@ -43,14 +43,16 @@ base64 = "0.21.5"
futures-util = "0.3.30"
pin-project = "1.1.5"
flowy-storage-pub = { workspace = true }
-collab-integrate.workspace = true
ollama-rs = "0.3.0"
-
+schemars = "0.8.22"
+twox-hash = { version = "2.1.0", features = ["xxhash64"] }
[target.'cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))'.dependencies]
notify = "6.1.1"
-#faiss = { version = "0.12.1" }
af-mcp = { version = "0.1.0" }
+text-splitter = { version = "0.25.1" }
+flowy-sqlite-vec.workspace = true
+
[dev-dependencies]
uuid.workspace = true
@@ -58,6 +60,7 @@ uuid.workspace = true
[build-dependencies]
flowy-codegen.workspace = true
+
[features]
dart = ["flowy-codegen/dart", "flowy-notification/dart"]
-local_ai = []
\ No newline at end of file
+local_ai = []
diff --git a/frontend/rust-lib/flowy-ai/src/ai_manager.rs b/frontend/rust-lib/flowy-ai/src/ai_manager.rs
index 9996947ed8..3d59a92b6a 100644
--- a/frontend/rust-lib/flowy-ai/src/ai_manager.rs
+++ b/frontend/rust-lib/flowy-ai/src/ai_manager.rs
@@ -17,10 +17,10 @@ use flowy_sqlite::kv::KVStorePreferences;
use crate::notification::{chat_notification_builder, ChatNotification};
use crate::util::ai_available_models_key;
-use collab_integrate::persistence::collab_metadata_sql::{
+use flowy_ai_pub::cloud::ai_dto::AvailableModel;
+use flowy_ai_pub::persistence::{
batch_insert_collab_metadata, batch_select_collab_metadata, AFCollabMetadata,
};
-use flowy_ai_pub::cloud::ai_dto::AvailableModel;
use flowy_ai_pub::user_service::AIUserService;
use flowy_storage_pub::storage::StorageService;
use lib_infra::async_trait::async_trait;
@@ -366,6 +366,7 @@ impl AIManager {
}
if need_restart {
+ self.local_ai.reload_ollama_client();
self.local_ai.restart_plugin().await;
}
@@ -497,17 +498,22 @@ impl AIManager {
match model {
None => {
- if let Some(local_model) = self.local_ai.get_plugin_chat_model() {
- model = Some(AIModel::local(local_model, "".to_string()));
- }
+ model = self
+ .store_preferences
+ .get_object::<AIModel>(&ai_available_models_key(GLOBAL_ACTIVE_MODEL_KEY));
+
model
},
Some(mut model) => {
- let models = self.local_ai.get_all_chat_local_models().await;
- if !models.contains(&model) {
- if let Some(local_model) = self.local_ai.get_plugin_chat_model() {
- model = AIModel::local(local_model, "".to_string());
- }
+ let mut all_models = vec![];
+ if let Ok(m) = self.get_server_available_models().await {
+ all_models.extend(m.into_iter().map(AIModel::from));
+ }
+
+ let local_models = self.local_ai.get_all_chat_local_models().await;
+ all_models.extend(local_models);
+ if !all_models.contains(&model) {
+ model = AIModel::default()
}
Some(model)
},
@@ -555,7 +561,15 @@ impl AIManager {
let setting = self.local_ai.get_local_ai_setting();
all_models.push(AIModel::local(setting.chat_model_name, "".to_string()));
} else {
- all_models.extend(self.local_ai.get_all_chat_local_models().await);
+ let local_models = self.local_ai.get_all_chat_local_models().await;
+ trace!(
+ "[Model Selection]: Available Local models: {:?}",
+ local_models
+ .iter()
+ .map(|m| m.name.as_str())
+ .collect::<Vec<&str>>()
+ );
+ all_models.extend(local_models);
}
}
@@ -605,6 +619,14 @@ impl AIManager {
},
};
+ trace!(
+ "[Model Selection] all models: {:?}",
+ all_models
+ .iter()
+ .map(|m| m.name.as_str())
+ .collect::<Vec<&str>>()
+ );
+
// Determine final active model - use user's selection if available, otherwise default
let active_model = all_models
.iter()
@@ -662,7 +684,6 @@ impl AIManager {
/// - `before_message_id` is the first message ID in the current chat messages.
///
/// 4. `after_message_id` and `before_message_id` cannot be specified at the same time.
-
pub async fn load_prev_chat_messages(
&self,
chat_id: &Uuid,
diff --git a/frontend/rust-lib/flowy-ai/src/embeddings/context.rs b/frontend/rust-lib/flowy-ai/src/embeddings/context.rs
new file mode 100644
index 0000000000..e1d9162502
--- /dev/null
+++ b/frontend/rust-lib/flowy-ai/src/embeddings/context.rs
@@ -0,0 +1,83 @@
+use crate::embeddings::scheduler::EmbeddingScheduler;
+use arc_swap::ArcSwapOption;
+use flowy_error::{ErrorCode, FlowyError, FlowyResult};
+use flowy_sqlite_vec::db::VectorSqliteDB;
+use lib_infra::util::get_operating_system;
+use ollama_rs::Ollama;
+use std::path::PathBuf;
+use std::sync::{Arc, OnceLock};
+use tracing::{error, info, warn};
+
+pub struct EmbedContext {
+ ollama: ArcSwapOption<Ollama>,
+ vector_db: ArcSwapOption<VectorSqliteDB>,
+ scheduler: ArcSwapOption<EmbeddingScheduler>,
+}
+
+impl EmbedContext {
+ pub fn shared() -> &'static Arc<EmbedContext> {
+ static INSTANCE: OnceLock<Arc<EmbedContext>> = OnceLock::new();
+ INSTANCE.get_or_init(|| {
+ Arc::new(EmbedContext {
+ ollama: ArcSwapOption::empty(),
+ vector_db: ArcSwapOption::empty(),
+ scheduler: Default::default(),
+ })
+ })
+ }
+
+ pub fn init_vector_db(&self, db_path: PathBuf) {
+ let sys = get_operating_system();
+ if !sys.is_desktop() {
+ warn!("[Embedding] Vector db is not supported on {:?}", sys);
+ return;
+ }
+
+ info!("Initializing vector db");
+ match VectorSqliteDB::new(db_path.clone()) {
+ Ok(db) => {
+ info!("[Embedding] Vector db created at: {:?}", db_path);
+ self.vector_db.store(Some(Arc::new(db)));
+ self.try_create_scheduler();
+ },
+ Err(err) => {
+ error!("[Embedding] Failed to create vector db: {}", err);
+ },
+ }
+ }
+
+ pub fn set_ollama(&self, ollama: Option<Arc<Ollama>>) {
+ if let Some(ollama) = ollama {
+ self.ollama.store(Some(ollama));
+ self.try_create_scheduler();
+ } else {
+ self.ollama.store(None);
+ if let Some(s) = self.scheduler.swap(None) {
+ info!("[Embedding] Stopping scheduler");
+ let _ = s.stop_tx.send(());
+ }
+ }
+ }
+
+ pub fn get_scheduler(&self) -> FlowyResult<Arc<EmbeddingScheduler>> {
+ self.scheduler.load_full().ok_or_else(|| {
+ FlowyError::new(
+ ErrorCode::LocalEmbeddingNotReady,
+ "Local embedding is not ready. Please check if the Ollama and vector db are initialized.",
+ )
+ })
+ }
+
+ fn try_create_scheduler(&self) {
+ if let (Some(ollama), Some(vector_db)) = (self.ollama.load_full(), self.vector_db.load_full()) {
+ info!("[Embedding] Creating scheduler");
+ match EmbeddingScheduler::new(ollama, vector_db) {
+ Ok(s) => {
+ info!("[Embedding] create scheduler successfully");
+ self.scheduler.store(Some(s));
+ },
+ Err(err) => error!("[Embedding] Failed to create scheduler: {}", err),
+ }
+ }
+ }
+}
diff --git a/frontend/rust-lib/flowy-ai/src/embeddings/document_indexer.rs b/frontend/rust-lib/flowy-ai/src/embeddings/document_indexer.rs
new file mode 100644
index 0000000000..bedd055494
--- /dev/null
+++ b/frontend/rust-lib/flowy-ai/src/embeddings/document_indexer.rs
@@ -0,0 +1,190 @@
+use crate::embeddings::embedder::Embedder;
+use crate::embeddings::indexer::{EmbeddingModel, Indexer};
+use flowy_ai_pub::entities::EmbeddedChunk;
+use flowy_error::FlowyError;
+use lib_infra::async_trait::async_trait;
+use ollama_rs::generation::embeddings::request::{EmbeddingsInput, GenerateEmbeddingsRequest};
+use serde_json::json;
+use text_splitter::{ChunkConfig, TextSplitter};
+use tracing::{debug, error, trace, warn};
+use twox_hash::xxhash64::Hasher;
+use uuid::Uuid;
+
+pub struct DocumentIndexer;
+
+#[async_trait]
+impl Indexer for DocumentIndexer {
+ fn create_embedded_chunks_from_text(
+ &self,
+ object_id: Uuid,
+ paragraphs: Vec<String>,
+ model: EmbeddingModel,
+ ) -> Result<Vec<EmbeddedChunk>, FlowyError> {
+ if paragraphs.is_empty() {
+ warn!(
+ "[Embedding] No paragraphs found in document `{}`. Skipping embedding.",
+ object_id
+ );
+
+ return Ok(vec![]);
+ }
+ split_text_into_chunks(&object_id.to_string(), paragraphs, model, 1000, 200)
+ }
+
+ async fn embed(
+ &self,
+ embedder: &Embedder,
+ mut chunks: Vec<EmbeddedChunk>,
+ ) -> Result<Vec<EmbeddedChunk>, FlowyError> {
+ let mut valid_indices = Vec::new();
+ for (i, chunk) in chunks.iter().enumerate() {
+ if let Some(ref content) = chunk.content {
+ if !content.is_empty() {
+ valid_indices.push(i);
+ }
+ }
+ }
+
+ if valid_indices.is_empty() {
+ return Ok(vec![]);
+ }
+
+ let mut contents = Vec::with_capacity(valid_indices.len());
+ for &i in &valid_indices {
+ contents.push(chunks[i].content.as_ref().unwrap().to_owned());
+ }
+
+ let request = GenerateEmbeddingsRequest::new(
+ embedder.model().name().to_string(),
+ EmbeddingsInput::Multiple(contents),
+ );
+ let resp = embedder.embed(request).await?;
+ if resp.embeddings.len() != valid_indices.len() {
+ error!(
+ "[Embedding] requested {} embeddings, received {} embeddings",
+ valid_indices.len(),
+ resp.embeddings.len()
+ );
+ return Err(FlowyError::internal().with_context(format!(
+ "Mismatch in number of embeddings requested and received: {} vs {}",
+ valid_indices.len(),
+ resp.embeddings.len()
+ )));
+ }
+
+ for (index, embedding) in resp.embeddings.into_iter().enumerate() {
+ let chunk_idx = valid_indices[index];
+ chunks[chunk_idx].embeddings = Some(embedding);
+ }
+
+ Ok(chunks)
+ }
+}
+
+/// chunk_size:
+/// Small Chunks (50–256 tokens): Best for precision-focused tasks (e.g., Q&A, technical docs) where specific details matter.
+/// Medium Chunks (256–1,024 tokens): Ideal for balanced tasks like RAG or contextual search, providing enough context without noise.
+/// Large Chunks (1,024–2,048 tokens): Suited for analysis or thematic tasks where broad understanding is key.
+///
+/// overlap:
+/// Add 10–20% overlap for larger chunks (e.g., 50–100 tokens for 512-token chunks) to preserve context across boundaries.
+pub fn split_text_into_chunks(
+ object_id: &str,
+ paragraphs: Vec<String>,
+ embedding_model: EmbeddingModel,
+ chunk_size: usize,
+ overlap: usize,
+) -> Result<Vec<EmbeddedChunk>, FlowyError> {
+ debug_assert!(matches!(embedding_model, EmbeddingModel::NomicEmbedText));
+
+ if paragraphs.is_empty() {
+ return Ok(vec![]);
+ }
+ let split_contents = group_paragraphs_by_max_content_len(paragraphs, chunk_size, overlap);
+ let metadata = json!({
+ "id": object_id,
+ "source": "appflowy",
+ "name": "document",
+ });
+
+ let mut seen = std::collections::HashSet::new();
+ let mut chunks = Vec::new();
+
+ for (index, content) in split_contents.into_iter().enumerate() {
+ let consistent_hash = Hasher::oneshot(0, content.as_bytes());
+ let fragment_id = format!("{:x}", consistent_hash);
+ if seen.insert(fragment_id.clone()) {
+ chunks.push(EmbeddedChunk {
+ fragment_id,
+ object_id: object_id.to_string(),
+ content_type: 0,
+ content: Some(content),
+ embeddings: None,
+ metadata: Some(metadata.to_string()),
+ fragment_index: index as i32,
+ embedder_type: 0,
+ });
+ } else {
+ debug!(
+ "[Embedding] Duplicate fragment_id detected: {}. This fragment will not be added.",
+ fragment_id
+ );
+ }
+ }
+
+ trace!(
+ "[Embedding] Created {} chunks for object_id `{}`, chunk_size: {}, overlap: {}",
+ chunks.len(),
+ object_id,
+ chunk_size,
+ overlap
+ );
+ Ok(chunks)
+}
+
+pub fn group_paragraphs_by_max_content_len(
+ paragraphs: Vec<String>,
+ mut context_size: usize,
+ overlap: usize,
+) -> Vec<String> {
+ if paragraphs.is_empty() {
+ return vec![];
+ }
+
+ let mut result = Vec::new();
+ let mut current = String::with_capacity(context_size.min(4096));
+
+ if overlap > context_size {
+ warn!("context_size is smaller than overlap, which may lead to unexpected behavior.");
+ context_size = 2 * overlap;
+ }
+
+ let chunk_config = ChunkConfig::new(context_size)
+ .with_overlap(overlap)
+ .unwrap();
+ let splitter = TextSplitter::new(chunk_config);
+
+ for paragraph in paragraphs {
+ if current.len() + paragraph.len() > context_size {
+ if !current.is_empty() {
+ result.push(std::mem::take(&mut current));
+ }
+
+ if paragraph.len() > context_size {
+ let paragraph_chunks = splitter.chunks(&paragraph);
+ result.extend(paragraph_chunks.map(String::from));
+ } else {
+ current.push_str(&paragraph);
+ }
+ } else {
+ // Add paragraph to current chunk
+ current.push_str(&paragraph);
+ }
+ }
+
+ if !current.is_empty() {
+ result.push(current);
+ }
+
+ result
+}
diff --git a/frontend/rust-lib/flowy-ai/src/embeddings/embedder.rs b/frontend/rust-lib/flowy-ai/src/embeddings/embedder.rs
new file mode 100644
index 0000000000..2a501ecda7
--- /dev/null
+++ b/frontend/rust-lib/flowy-ai/src/embeddings/embedder.rs
@@ -0,0 +1,41 @@
+use crate::embeddings::indexer::EmbeddingModel;
+use flowy_error::FlowyResult;
+use ollama_rs::generation::embeddings::request::GenerateEmbeddingsRequest;
+use ollama_rs::generation::embeddings::GenerateEmbeddingsResponse;
+use ollama_rs::Ollama;
+use std::sync::Arc;
+
+#[derive(Debug, Clone)]
+pub enum Embedder {
+ Ollama(OllamaEmbedder),
+}
+
+impl Embedder {
+ pub async fn embed(
+ &self,
+ request: GenerateEmbeddingsRequest,
+ ) -> FlowyResult<GenerateEmbeddingsResponse> {
+ match self {
+ Embedder::Ollama(ollama) => ollama.embed(request).await,
+ }
+ }
+
+ pub fn model(&self) -> EmbeddingModel {
+ EmbeddingModel::NomicEmbedText
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct OllamaEmbedder {
+ pub ollama: Arc<Ollama>,
+}
+
+impl OllamaEmbedder {
+ pub async fn embed(
+ &self,
+ request: GenerateEmbeddingsRequest,
+ ) -> FlowyResult<GenerateEmbeddingsResponse> {
+ let resp = self.ollama.generate_embeddings(request).await?;
+ Ok(resp)
+ }
+}
diff --git a/frontend/rust-lib/flowy-ai/src/embeddings/indexer.rs b/frontend/rust-lib/flowy-ai/src/embeddings/indexer.rs
new file mode 100644
index 0000000000..a7caa84f8b
--- /dev/null
+++ b/frontend/rust-lib/flowy-ai/src/embeddings/indexer.rs
@@ -0,0 +1,70 @@
+use crate::embeddings::document_indexer::DocumentIndexer;
+use crate::embeddings::embedder::Embedder;
+use flowy_ai_pub::cloud::CollabType;
+use flowy_ai_pub::entities::EmbeddedChunk;
+use flowy_error::FlowyError;
+use lib_infra::async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::Arc;
+use uuid::Uuid;
+
+#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub enum EmbeddingModel {
+ NomicEmbedText,
+}
+
+impl EmbeddingModel {
+ pub fn name(&self) -> &'static str {
+ match self {
+ EmbeddingModel::NomicEmbedText => "nomic-embed-text",
+ }
+ }
+
+ #[allow(dead_code)]
+ pub fn dimension(&self) -> usize {
+ match self {
+ // https://ollama.com/library/nomic-embed-text/blobs/970aa74c0a90
+ EmbeddingModel::NomicEmbedText => 768,
+ }
+ }
+}
+
+#[async_trait]
+pub trait Indexer: Send + Sync {
+ fn create_embedded_chunks_from_text(
+ &self,
+ object_id: Uuid,
+ paragraphs: Vec<String>,
+ model: EmbeddingModel,
+ ) -> Result<Vec<EmbeddedChunk>, FlowyError>;
+
+ async fn embed(
+ &self,
+ embedder: &Embedder,
+ chunks: Vec<EmbeddedChunk>,
+ ) -> Result<Vec<EmbeddedChunk>, FlowyError>;
+}
+
+/// A structure responsible for resolving different [Indexer] types for different [CollabType]s,
+/// including access permission checks for the specific workspaces.
+pub struct IndexerProvider {
+ indexer_cache: HashMap<CollabType, Arc<dyn Indexer>>,
+}
+
+impl IndexerProvider {
+ pub fn new() -> Arc<Self> {
+ let mut cache: HashMap<CollabType, Arc<dyn Indexer>> = HashMap::new();
+ cache.insert(CollabType::Document, Arc::new(DocumentIndexer));
+ Arc::new(Self {
+ indexer_cache: cache,
+ })
+ }
+
+ /// Returns indexer for a specific type of [Collab] object.
+ /// If collab of given type is not supported or workspace it belongs to has indexing disabled,
+ /// returns `None`.
+ pub fn indexer_for(&self, collab_type: CollabType) -> Option<Arc<dyn Indexer>> {
+ self.indexer_cache.get(&collab_type).cloned()
+ }
+}
diff --git a/frontend/rust-lib/flowy-ai/src/embeddings/mod.rs b/frontend/rust-lib/flowy-ai/src/embeddings/mod.rs
new file mode 100644
index 0000000000..abbe1cd443
--- /dev/null
+++ b/frontend/rust-lib/flowy-ai/src/embeddings/mod.rs
@@ -0,0 +1,5 @@
+pub mod context;
+mod document_indexer;
+mod embedder;
+mod indexer;
+mod scheduler;
diff --git a/frontend/rust-lib/flowy-ai/src/embeddings/scheduler.rs b/frontend/rust-lib/flowy-ai/src/embeddings/scheduler.rs
new file mode 100644
index 0000000000..327ecda58d
--- /dev/null
+++ b/frontend/rust-lib/flowy-ai/src/embeddings/scheduler.rs
@@ -0,0 +1,346 @@
+use crate::embeddings::embedder::{Embedder, OllamaEmbedder};
+use crate::embeddings::indexer::IndexerProvider;
+use crate::search::summary::{summarize_documents, LLMDocument};
+use flowy_ai_pub::cloud::search_dto::{
+ SearchContentType, SearchDocumentResponseItem, SearchResult, SearchSummaryResult, Summary,
+};
+use flowy_ai_pub::entities::{EmbeddingRecord, UnindexedCollab, UnindexedData};
+use flowy_error::{ErrorCode, FlowyError, FlowyResult};
+use flowy_sqlite::internal::derives::multiconnection::chrono::Utc;
+use flowy_sqlite_vec::db::VectorSqliteDB;
+use ollama_rs::generation::embeddings::request::{EmbeddingsInput, GenerateEmbeddingsRequest};
+use ollama_rs::Ollama;
+use std::sync::{Arc, Weak};
+use tokio::select;
+use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
+use tokio::sync::{broadcast, mpsc};
+use tracing::{debug, error, info, trace, warn};
+use uuid::Uuid;
+
+type UnindexedCollabContext = UnindexedCollab;
+
+pub struct EmbeddingScheduler {
+ indexer_provider: Arc<IndexerProvider>,
+ write_embedding_tx: UnboundedSender<EmbeddingRecord>,
+ generate_embedding_tx: mpsc::Sender<UnindexedCollab>,
+ ollama: Arc<Ollama>,
+ vector_db: Arc<VectorSqliteDB>,
+ pub(crate) stop_tx: tokio::sync::broadcast::Sender<()>,
+}
+
+impl EmbeddingScheduler {
+ pub fn new(
+ ollama: Arc<Ollama>,
+ vector_db: Arc<VectorSqliteDB>,
+ ) -> FlowyResult<Arc<EmbeddingScheduler>> {
+ let indexer_provider = IndexerProvider::new();
+ let (write_embedding_tx, write_embedding_rx) = unbounded_channel::<EmbeddingRecord>();
+ let (generate_embedding_tx, gen_embedding_rx) = mpsc::channel::<UnindexedCollabContext>(100);
+ let (stop_tx, _) = broadcast::channel::<()>(1);
+
+ let this = Arc::new(Self {
+ indexer_provider,
+ write_embedding_tx,
+ generate_embedding_tx,
+ ollama,
+ vector_db,
+ stop_tx,
+ });
+
+ let weak_this = Arc::downgrade(&this);
+ let stop_rx = this.stop_tx.subscribe();
+ tokio::spawn(spawn_generate_embeddings(
+ gen_embedding_rx,
+ weak_this.clone(),
+ stop_rx,
+ ));
+
+ let weak_this = Arc::downgrade(&this);
+ let stop_rx = this.stop_tx.subscribe();
+ tokio::spawn(spawn_write_embeddings(
+ write_embedding_rx,
+ weak_this,
+ stop_rx,
+ ));
+
+ Ok(this)
+ }
+
+ pub(crate) fn create_embedder(&self) -> Result<Embedder, FlowyError> {
+ let embedder = Embedder::Ollama(OllamaEmbedder {
+ ollama: self.ollama.clone(),
+ });
+ Ok(embedder)
+ }
+
+ pub async fn index_collab(&self, data: UnindexedCollab) -> FlowyResult<()> {
+ trace!("[Embedding] got {} unindexd data", data.object_id);
+ if let Err(err) = self.generate_embedding_tx.send(data).await {
+ error!("[Embedding] error generating embedding: {}", err);
+ }
+ Ok(())
+ }
+
+ pub async fn delete_collab(&self, workspace_id: &Uuid, object_id: &Uuid) -> FlowyResult<()> {
+ self
+ .vector_db
+ .delete_collab(&workspace_id.to_string(), &object_id.to_string())
+ .await
+ .map_err(|err| {
+ error!("[Embedding] Failed to delete collab: {}", err);
+ FlowyError::new(ErrorCode::LocalEmbeddingNotReady, "Failed to delete collab")
+ })?;
+ Ok(())
+ }
+
+ pub async fn search(
+ &self,
+ workspace_id: &Uuid,
+ query: &str,
+ ) -> FlowyResult<Vec<SearchDocumentResponseItem>> {
+ let embedder = self.create_embedder()?;
+ let request = GenerateEmbeddingsRequest::new(
+ embedder.model().name().to_string(),
+ EmbeddingsInput::Single(query.to_string()),
+ );
+
+ let resp = embedder.embed(request).await?;
+ match resp.embeddings.first() {
+ None => Ok(vec![]),
+ Some(query_embed) => {
+ let result = self
+ .vector_db
+ .search_with_score(&workspace_id.to_string(), query_embed, 10, 0.4)
+ .await
+ .map_err(|err| {
+ error!("[Embedding] Failed to search: {}", err);
+ FlowyError::new(ErrorCode::LocalEmbeddingNotReady, "Failed to search")
+ })?;
+
+ let rows = result
+ .into_iter()
+ .map(|v| SearchDocumentResponseItem {
+ object_id: v.oid,
+ workspace_id: *workspace_id,
+ score: 1.0,
+ content_type: Some(SearchContentType::PlainText),
+ content: v.content,
+ preview: None,
+ created_by: "".to_string(),
+ created_at: Utc::now(),
+ })
+ .collect::<Vec<_>>();
+
+ Ok(rows)
+ },
+ }
+ }
+
+ pub async fn generate_summary(
+ &self,
+ question: &str,
+ model_name: &str,
+ search_results: Vec<SearchResult>,
+ ) -> FlowyResult<SearchSummaryResult> {
+ if search_results.is_empty() {
+ return Ok(SearchSummaryResult { summaries: vec![] });
+ }
+
+ let docs = search_results
+ .into_iter()
+ .map(|v| LLMDocument {
+ content: v.content,
+ object_id: v.object_id,
+ })
+ .collect::<Vec<_>>();
+ let resp = summarize_documents(&self.ollama, question, model_name, docs)
+ .await
+ .map_err(|err| {
+ error!("[Embedding] Failed to generate summary: {}", err);
+ FlowyError::new(
+ ErrorCode::LocalEmbeddingNotReady,
+ "Failed to generate summary",
+ )
+ })?;
+
+ let summaries = resp
+ .summaries
+ .into_iter()
+ .flat_map(|s| {
+ if s.content.is_empty() {
+ None
+ } else {
+ Some(Summary {
+ content: s.content,
+ highlights: s.highlights,
+ sources: s.sources,
+ })
+ }
+ })
+ .collect::<Vec<_>>();
+ Ok(SearchSummaryResult { summaries })
+ }
+}
+
+const EMBEDDING_RECORD_BUFFER_SIZE: usize = 10;
+
+pub async fn spawn_write_embeddings(
+ mut rx: UnboundedReceiver<EmbeddingRecord>,
+ scheduler: Weak<EmbeddingScheduler>,
+ mut stop_rx: broadcast::Receiver<()>,
+) {
+ let mut buf = Vec::with_capacity(EMBEDDING_RECORD_BUFFER_SIZE);
+
+ loop {
+ select! {
+ // Shutdown signal arrives
+ _ = stop_rx.recv() => {
+ info!("Received stop signal; shutting down embedding writer");
+ break;
+ }
+ // Next batch from the input channel
+ n = rx.recv_many(&mut buf, EMBEDDING_RECORD_BUFFER_SIZE) => {
+ // channel closed
+ if n == 0 {
+ info!("Input channel closed; stopping write embeddings");
+ break;
+ }
+
+ // upgrade scheduler reference
+ let scheduler = match scheduler.upgrade() {
+ Some(db) => db,
+ None => {
+ error!("EmbeddingScheduler dropped; stopping write embeddings");
+ break;
+ }
+ };
+
+ // drain and process exactly `n` records
+ let records = buf.drain(..n).collect::<Vec<_>>();
+ for record in records {
+ debug!("[Embedding] Writing {} chunks for {}", record.chunks.len(), record.object_id);
+ match scheduler
+ .vector_db
+ .upsert_collabs_embeddings(&record.workspace_id.to_string(), &record.object_id.to_string(), record.chunks)
+ .await
+ {
+ Ok(_) => trace!("[Embedding] Successfully wrote embeddings for {}", record.object_id),
+ Err(err) => error!("[Embedding] Failed to write embeddings for {}: {}", record.object_id, err),
+ }
+ }
+ }
+ }
+ }
+
+ info!("spawn_write_embeddings exited");
+}
+
+async fn spawn_generate_embeddings(
+ mut rx: mpsc::Receiver<UnindexedCollab>,
+ scheduler: Weak<EmbeddingScheduler>,
+ mut stop_rx: broadcast::Receiver<()>,
+) {
+ let mut buf = Vec::with_capacity(EMBEDDING_RECORD_BUFFER_SIZE);
+ loop {
+ select! {
+ _ = stop_rx.recv() => {
+ info!("Received stop signal; shutting down embedding writer");
+ break;
+ }
+ n = rx.recv_many(&mut buf, EMBEDDING_RECORD_BUFFER_SIZE) => {
+ let scheduler = match scheduler.upgrade() {
+ Some(scheduler) => scheduler,
+ None => {
+ info!("[Embedding] Failed to upgrade scheduler connection, break loop");
+ break;
+ },
+ };
+ if n == 0 {
+ info!("[Embedding] Stop generating embeddings");
+ break;
+ }
+
+ let records = buf.drain(..n).collect::<Vec<_>>();
+ let indexer_provider = scheduler.indexer_provider.clone();
+ let write_embedding_tx = scheduler.write_embedding_tx.clone();
+ let embedder = scheduler.create_embedder();
+
+ match embedder {
+ Ok(embedder) => {
+ let params: Vec<_> = records.iter().map(|r| r.object_id.to_string()).collect();
+ let existing_embeddings = scheduler
+ .vector_db
+ .select_collabs_fragment_ids(&params)
+ .await
+ .unwrap_or_else(|err| {
+ error!("[Embedding] failed to get existing embeddings: {}", err);
+ Default::default()
+ });
+
+ for record in records {
+ if let Some(indexer) = indexer_provider.indexer_for(record.collab_type) {
+ let paragraphs = match record.data {
+ UnindexedData::Paragraphs(paragraphs) => paragraphs,
+ UnindexedData::Text(text) => text.split('\n').map(|s| s.to_string()).collect(),
+ };
+ let embedder = embedder.clone();
+ match indexer.create_embedded_chunks_from_text(
+ record.object_id,
+ paragraphs,
+ embedder.model(),
+ ) {
+ Ok(mut chunks) => {
+ if let Some(fragment_ids) = existing_embeddings.get(&record.object_id) {
+ for chunk in chunks.iter_mut() {
+ if fragment_ids.contains(&chunk.fragment_id) {
+ chunk.content = None;
+ }
+ }
+ }
+
+ if chunks.iter().all(|c| c.content.is_none()) {
+ trace!(
+ "[Embedding] skip generating embeddings for collab: {}",
+ record.object_id
+ );
+ continue;
+ }
+
+ let result = indexer.embed(&embedder, chunks).await;
+ match result {
+ Ok(chunks) => {
+ let record = EmbeddingRecord {
+ workspace_id: record.workspace_id,
+ object_id: record.object_id,
+ chunks,
+ };
+ if let Err(err) = write_embedding_tx.send(record) {
+ error!("Failed to send embedding record: {}", err);
+ }
+ },
+ Err(err) => {
+ error!(
+ "[Embedding] Failed to create embeddings content for collab: {}, error:{}",
+ record.object_id, err
+ );
+ },
+ }
+ },
+ Err(err) => {
+ warn!(
+ "Failed to create embedded chunks for collab: {}, error:{}",
+ record.object_id, err
+ );
+ continue;
+ },
+ }
+ }
+ }
+ },
+ Err(err) => error!("[Embedding] Failed to create embedder: {}", err),
+ }
+ }
+ }
+ }
+ info!("spawn_generate_embeddings exited");
+}
diff --git a/frontend/rust-lib/flowy-ai/src/lib.rs b/frontend/rust-lib/flowy-ai/src/lib.rs
index 5b582b2577..e0c9d1ac89 100644
--- a/frontend/rust-lib/flowy-ai/src/lib.rs
+++ b/frontend/rust-lib/flowy-ai/src/lib.rs
@@ -10,9 +10,12 @@ pub mod local_ai;
// #[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))]
// pub mod mcp;
+#[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))]
+pub mod embeddings;
mod middleware;
pub mod notification;
pub mod offline;
mod protobuf;
+mod search;
mod stream_message;
mod util;
diff --git a/frontend/rust-lib/flowy-ai/src/local_ai/controller.rs b/frontend/rust-lib/flowy-ai/src/local_ai/controller.rs
index 6b147b9c0c..878e2eaa23 100644
--- a/frontend/rust-lib/flowy-ai/src/local_ai/controller.rs
+++ b/frontend/rust-lib/flowy-ai/src/local_ai/controller.rs
@@ -77,7 +77,7 @@ impl LocalAIController {
user_service: Arc<dyn AIUserService>,
) -> Self {
debug!(
- "[AI Plugin] init local ai controller, thread: {:?}",
+ "[Local AI] init local ai controller, thread: {:?}",
std::thread::current().id()
);
@@ -92,15 +92,9 @@ impl LocalAIController {
));
let ollama = ArcSwapOption::default();
+
let sys = get_operating_system();
if sys.is_desktop() {
- let setting = local_ai_resource.get_llm_setting();
- ollama.store(
- Ollama::try_new(&setting.ollama_server_url)
- .map(Arc::new)
- .ok(),
- );
-
// Subscribe to state changes
let mut running_state_rx = local_ai.subscribe_running_state();
let cloned_llm_res = Arc::clone(&local_ai_resource);
@@ -116,8 +110,8 @@ impl LocalAIController {
continue;
};
- let key = crate::local_ai::controller::local_ai_enabled_key(&workspace_id.to_string());
- info!("[AI Plugin] state: {:?}", state);
+ let key = local_ai_enabled_key(&workspace_id.to_string());
+ info!("[Local AI] state: {:?}", state);
// Read whether plugin is enabled from store; default to true
if let Some(store_preferences) = cloned_store_preferences.upgrade() {
@@ -159,7 +153,7 @@ impl LocalAIController {
})
.send();
} else {
- warn!("[AI Plugin] store preferences is dropped");
+ warn!("[Local AI] store preferences is dropped");
}
}
});
@@ -174,6 +168,36 @@ impl LocalAIController {
ollama,
}
}
+
+ pub fn reload_ollama_client(&self) {
+ if !self.is_enabled() {
+ return;
+ }
+
+ let setting = self.resource.get_llm_setting();
+ if let Some(ollama) = self.ollama.load_full() {
+ if ollama.url_str() == setting.ollama_server_url {
+ info!("[Local AI] ollama client is already initialized");
+ return;
+ }
+ }
+
+ info!("[Local AI] reloading ollama client");
+ match Ollama::try_new(setting.ollama_server_url).map(Arc::new) {
+ Ok(new_ollama) => {
+ self.ollama.store(Some(new_ollama.clone()));
+
+ #[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))]
+ crate::embeddings::context::EmbedContext::shared().set_ollama(Some(new_ollama.clone()));
+ },
+ Err(err) => error!(
+ "failed to create ollama client: {:?}, thread: {:?}",
+ err,
+ std::thread::current().id()
+ ),
+ }
+ }
+
#[instrument(level = "debug", skip_all)]
pub async fn observe_plugin_resource(&self) {
let sys = get_operating_system();
@@ -182,7 +206,7 @@ impl LocalAIController {
}
debug!(
- "[AI Plugin] observer plugin state. thread: {:?}",
+ "[Local AI] observer plugin state. thread: {:?}",
std::thread::current().id()
);
async fn try_init_plugin(
@@ -190,7 +214,7 @@ impl LocalAIController {
ai_plugin: &Arc<OllamaAIPlugin>,
) {
if let Err(err) = initialize_ai_plugin(ai_plugin, resource, None).await {
- error!("[AI Plugin] failed to setup plugin: {:?}", err);
+ error!("[Local AI] failed to setup plugin: {:?}", err);
}
}
@@ -203,11 +227,11 @@ impl LocalAIController {
loop {
select! {
_ = app_state_watcher.recv() => {
- info!("[AI Plugin] app state changed, try to init plugin");
+ info!("[Local AI] app state changed, try to init plugin");
try_init_plugin(&resource_clone, &ai_plugin_clone).await;
},
_ = resource_notify.recv() => {
- info!("[AI Plugin] resource changed, try to init plugin");
+ info!("[Local AI] resource changed, try to init plugin");
try_init_plugin(&resource_clone, &ai_plugin_clone).await;
},
else => break,
@@ -270,7 +294,7 @@ impl LocalAIController {
// Only keep one chat open at a time. Since loading multiple models at the same time will cause
// memory issues.
if let Some(current_chat_id) = self.current_chat_id.load().as_ref() {
- debug!("[AI Plugin] close previous chat: {}", current_chat_id);
+ debug!("[Local AI] close previous chat: {}", current_chat_id);
self.close_chat(current_chat_id);
}
@@ -280,7 +304,7 @@ impl LocalAIController {
tokio::spawn(async move {
if let Some(ctrl) = weak_ctrl.upgrade() {
if let Err(err) = ctrl.create_chat(&chat_id).await {
- error!("[AI Plugin] failed to open chat: {:?}", err);
+ error!("[Local AI] failed to open chat: {:?}", err);
}
}
});
@@ -290,13 +314,13 @@ impl LocalAIController {
if !self.is_running() {
return;
}
- info!("[AI Plugin] notify close chat: {}", chat_id);
+ info!("[Local AI] notify close chat: {}", chat_id);
let weak_ctrl = Arc::downgrade(&self.ai_plugin);
let chat_id = chat_id.to_string();
tokio::spawn(async move {
if let Some(ctrl) = weak_ctrl.upgrade() {
if let Err(err) = ctrl.close_chat(&chat_id).await {
- error!("[AI Plugin] failed to close chat: {:?}", err);
+ error!("[Local AI] failed to close chat: {:?}", err);
}
}
});
@@ -380,7 +404,7 @@ impl LocalAIController {
pub async fn update_local_ai_setting(&self, setting: LocalAISetting) -> FlowyResult<()> {
info!(
- "[AI Plugin] update local ai setting: {:?}, thread: {:?}",
+ "[Local AI] update local ai setting: {:?}, thread: {:?}",
setting,
std::thread::current().id()
);
@@ -396,7 +420,7 @@ impl LocalAIController {
// If not enabled, return immediately.
if !enabled {
debug!(
- "[AI Plugin] get local ai state, elapsed: {:?}, thread: {:?}",
+ "[Local AI] get local ai state, elapsed: {:?}, thread: {:?}",
start.elapsed(),
std::thread::current().id()
);
@@ -429,7 +453,7 @@ impl LocalAIController {
let elapsed = start.elapsed();
debug!(
- "[AI Plugin] get local ai state, elapsed: {:?}, thread: {:?}",
+ "[Local AI] get local ai state, elapsed: {:?}, thread: {:?}",
elapsed,
std::thread::current().id()
);
@@ -445,7 +469,7 @@ impl LocalAIController {
#[instrument(level = "debug", skip_all)]
pub async fn restart_plugin(&self) {
if let Err(err) = initialize_ai_plugin(&self.ai_plugin, &self.resource, None).await {
- error!("[AI Plugin] failed to setup plugin: {:?}", err);
+ error!("[Local AI] failed to setup plugin: {:?}", err);
}
}
@@ -461,7 +485,7 @@ impl LocalAIController {
let key = local_ai_enabled_key(&workspace_id.to_string());
let store_preferences = self.upgrade_store_preferences()?;
let enabled = !store_preferences.get_bool(&key).unwrap_or(false);
- tracing::trace!("[AI Plugin] toggle local ai, enabled: {}", enabled,);
+ tracing::trace!("[Local AI] toggle local ai, enabled: {}", enabled,);
store_preferences.set_bool(&key, enabled)?;
self.toggle_plugin(enabled).await?;
Ok(enabled)
@@ -475,7 +499,7 @@ impl LocalAIController {
// index_process_sink: &mut (impl Sink<String> + Unpin),
// ) -> FlowyResult<()> {
// if !self.is_enabled() {
- // info!("[AI Plugin] local ai is disabled, skip indexing");
+ // info!("[Local AI] local ai is disabled, skip indexing");
// return Ok(());
// }
//
@@ -492,14 +516,14 @@ impl LocalAIController {
// );
// }
// info!(
- // "[AI Plugin] embed file: {:?}, with metadata: {:?}",
+ // "[Local AI] embed file: {:?}, with metadata: {:?}",
// file_path, file_metadata
// );
//
// match &metadata.data.content_type {
// ContextLoader::Unknown => {
// error!(
- // "[AI Plugin] unsupported content type: {:?}",
+ // "[Local AI] unsupported content type: {:?}",
// metadata.data.content_type
// );
// },
@@ -560,7 +584,7 @@ impl LocalAIController {
let _ = index_process_sink
.send(StreamMessage::IndexFileError { file_name }.to_string())
.await;
- error!("[AI Plugin] failed to index file: {:?}", err);
+ error!("[Local AI] failed to index file: {:?}", err);
},
}
@@ -570,19 +594,19 @@ impl LocalAIController {
#[instrument(level = "debug", skip_all)]
pub(crate) async fn toggle_plugin(&self, enabled: bool) -> FlowyResult<()> {
info!(
- "[AI Plugin] enable: {}, thread id: {:?}",
+ "[Local AI] enable: {}, thread id: {:?}",
enabled,
std::thread::current().id()
);
if enabled {
let (tx, rx) = tokio::sync::oneshot::channel();
if let Err(err) = initialize_ai_plugin(&self.ai_plugin, &self.resource, Some(tx)).await {
- error!("[AI Plugin] failed to initialize local ai: {:?}", err);
+ error!("[Local AI] failed to initialize local ai: {:?}", err);
}
let _ = rx.await;
} else {
if let Err(err) = self.ai_plugin.destroy_plugin().await {
- error!("[AI Plugin] failed to destroy plugin: {:?}", err);
+ error!("[Local AI] failed to destroy plugin: {:?}", err);
}
chat_notification_builder(
@@ -625,7 +649,7 @@ async fn initialize_ai_plugin(
if let Some(lack_of_resource) = lack_of_resource {
info!(
- "[AI Plugin] lack of resource: {:?} to initialize plugin, thread: {:?}",
+ "[Local AI] lack of resource: {:?} to initialize plugin, thread: {:?}",
lack_of_resource,
std::thread::current().id()
);
@@ -641,7 +665,7 @@ async fn initialize_ai_plugin(
if let Err(err) = plugin.destroy_plugin().await {
error!(
- "[AI Plugin] failed to destroy plugin when lack of resource: {:?}",
+ "[Local AI] failed to destroy plugin when lack of resource: {:?}",
err
);
}
@@ -653,14 +677,14 @@ async fn initialize_ai_plugin(
match cloned_llm_res.get_plugin_config(true).await {
Ok(config) => {
info!(
- "[AI Plugin] initialize plugin with config: {:?}, thread: {:?}",
+ "[Local AI] initialize plugin with config: {:?}, thread: {:?}",
config,
std::thread::current().id()
);
match plugin.init_plugin(config).await {
Ok(_) => {},
- Err(err) => error!("[AI Plugin] failed to setup plugin: {:?}", err),
+ Err(err) => error!("[Local AI] failed to setup plugin: {:?}", err),
}
if let Some(ret) = ret {
@@ -668,7 +692,7 @@ async fn initialize_ai_plugin(
}
},
Err(err) => {
- error!("[AI Plugin] failed to get plugin config: {:?}", err);
+ error!("[Local AI] failed to get plugin config: {:?}", err);
},
};
})
diff --git a/frontend/rust-lib/flowy-ai/src/search/mod.rs b/frontend/rust-lib/flowy-ai/src/search/mod.rs
new file mode 100644
index 0000000000..1c631a9922
--- /dev/null
+++ b/frontend/rust-lib/flowy-ai/src/search/mod.rs
@@ -0,0 +1 @@
+pub mod summary;
diff --git a/frontend/rust-lib/flowy-ai/src/search/summary.rs b/frontend/rust-lib/flowy-ai/src/search/summary.rs
new file mode 100644
index 0000000000..1b38b9980f
--- /dev/null
+++ b/frontend/rust-lib/flowy-ai/src/search/summary.rs
@@ -0,0 +1,123 @@
+use flowy_error::FlowyError;
+use ollama_rs::generation::chat::request::ChatMessageRequest;
+use ollama_rs::generation::chat::{ChatMessage, MessageRole};
+use ollama_rs::generation::parameters::{FormatType, JsonStructure};
+use ollama_rs::Ollama;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+use tracing::error;
+use uuid::Uuid;
+
+#[derive(Debug)]
+pub struct SearchSummary {
+ pub content: String,
+ pub highlights: String,
+ pub sources: Vec<Uuid>,
+}
+
+#[derive(Debug)]
+pub struct SummarySearchResponse {
+ pub summaries: Vec<SearchSummary>,
+}
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema)]
+#[serde(deny_unknown_fields)]
+struct SummarySearchSchema {
+ pub answer: String,
+ pub highlights: String,
+ pub sources: Vec<String>,
+}
+
+const SYSTEM_PROMPT: &str = r#"
+You are a strict, context-bound question answering assistant. Answer solely based on the context provided below. If the context lacks sufficient information for a confident response, reply with an empty answer.
+Do not reference or use any information beyond what is provided in the context.
+
+Output must include:
+- `answer`: a detailed, on-point answer to the user’s question.
+- `highlights`:A markdown bullet list that highlights key themes and important details (e.g., date, time, location, etc.).
+- `sources`: array of source IDs used for the answer.
+"#;
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct LLMDocument {
+ pub content: String,
+ pub object_id: Uuid,
+}
+
+impl LLMDocument {
+ #[allow(dead_code)]
+ pub fn new(content: String, object_id: Uuid) -> Self {
+ Self { content, object_id }
+ }
+}
+
+fn convert_documents_to_text(documents: Vec<LLMDocument>) -> String {
+ documents
+ .into_iter()
+ .map(|doc| json!(doc).to_string())
+ .collect::<Vec<String>>()
+ .join("\n")
+}
+
+pub async fn summarize_documents(
+ client: &Ollama,
+ question: &str,
+ model_name: &str,
+ documents: Vec<LLMDocument>,
+) -> Result<SummarySearchResponse, FlowyError> {
+ let documents_text = convert_documents_to_text(documents);
+ let context = format!("{}\n\n##Context##\n{}", SYSTEM_PROMPT, documents_text);
+ let messages = vec![
+ ChatMessage::new(MessageRole::System, context),
+ ChatMessage::new(MessageRole::User, question.to_string()),
+ ];
+
+ let format = FormatType::StructuredJson(JsonStructure::new::<SummarySearchSchema>());
+ let request = ChatMessageRequest::new(model_name.to_string(), messages).format(format);
+ match client.send_chat_messages(request).await {
+ Ok(resp) => {
+ if resp.final_data.is_some() {
+ let resp: SummarySearchSchema = serde_json::from_str(&resp.message.content)?;
+ let resp = SummarySearchResponse {
+ summaries: vec![SearchSummary {
+ content: resp.answer,
+ highlights: resp.highlights,
+ sources: resp
+ .sources
+ .into_iter()
+ .flat_map(|s| Uuid::parse_str(&s).ok())
+ .collect(),
+ }],
+ };
+ Ok(resp)
+ } else {
+ Ok(SummarySearchResponse { summaries: vec![] })
+ }
+ },
+ Err(err) => {
+ error!("Error generating summary: {}", err);
+ Ok(SummarySearchResponse { summaries: vec![] })
+ },
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::search::summary::{summarize_documents, LLMDocument};
+ use ollama_rs::Ollama;
+
+ #[tokio::test]
+ async fn summarize_documents_test() {
+ let ollama = Ollama::try_new("http://localhost:11434").unwrap();
+ let docs = vec![
+ ("Rust is a multiplayer survival game developed by Facepunch Studios, first released in early access in December 2013 and fully launched in February 2018. It has since become one of the most popular games in the survival genre, known for its harsh environment, intricate crafting system, and player-driven dynamics. The game is available on Windows, macOS, and PlayStation, with a community-driven approach to updates and content additions.", uuid::Uuid::new_v4()),
+ ("Rust is a modern, system-level programming language designed with a focus on performance, safety, and concurrency. It was created by Mozilla and first released in 2010, with its 1.0 version launched in 2015. Rust is known for providing the control and performance of languages like C and C++, but with built-in safety features that prevent common programming errors, such as memory leaks, data races, and buffer overflows.", uuid::Uuid::new_v4()),
+ ("Rust as a Natural Process (Oxidation) refers to the chemical reaction that occurs when metals, primarily iron, come into contact with oxygen and moisture (water) over time, leading to the formation of iron oxide, commonly known as rust. This process is a form of oxidation, where a substance reacts with oxygen in the air or water, resulting in the degradation of the metal.", uuid::Uuid::new_v4()),
+ ].into_iter().map(|(content, object_id)| LLMDocument::new(content.to_string(), object_id)).collect::<Vec<_>>();
+ let result = summarize_documents(&ollama, "multiplayer game", "llama3.1", docs.clone())
+ .await
+ .unwrap();
+ dbg!(result);
+ }
+}
diff --git a/frontend/rust-lib/flowy-core/Cargo.toml b/frontend/rust-lib/flowy-core/Cargo.toml
index b4e7bd5fec..647cde1ac8 100644
--- a/frontend/rust-lib/flowy-core/Cargo.toml
+++ b/frontend/rust-lib/flowy-core/Cargo.toml
@@ -18,7 +18,7 @@ flowy-sqlite = { workspace = true }
flowy-document = { workspace = true }
flowy-document-pub = { workspace = true }
flowy-error = { workspace = true }
-flowy-server = { workspace = true, features = ["enable_supabase"] }
+flowy-server = { workspace = true }
flowy-server-pub = { workspace = true }
flowy-date = { workspace = true }
collab-integrate = { workspace = true }
@@ -27,6 +27,7 @@ flowy-search-pub = { workspace = true }
collab-entity = { workspace = true }
collab-plugins = { workspace = true }
collab-folder = { workspace = true }
+collab-document = { workspace = true }
collab = { workspace = true }
#collab = { workspace = true, features = ["verbose_log"] }
@@ -51,15 +52,20 @@ anyhow.workspace = true
dashmap.workspace = true
arc-swap.workspace = true
base64 = "0.21.5"
-
+tokio-util.workspace = true
lib-infra = { workspace = true }
serde.workspace = true
serde_json.workspace = true
-serde_repr.workspace = true
uuid.workspace = true
sysinfo = "0.30.5"
semver = { version = "1.0.22", features = ["serde"] }
url = "2.5.0"
+once_cell = "1.19.0"
+
+
+[target.'cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))'.dependencies]
+flowy-sqlite-vec = { workspace = true }
+twox-hash = { version = "2.1.0", features = ["xxhash64"] }
[features]
profiling = ["console-subscriber", "tokio/tracing"]
diff --git a/frontend/rust-lib/flowy-core/src/user_state_callback.rs b/frontend/rust-lib/flowy-core/src/app_life_cycle.rs
similarity index 60%
rename from frontend/rust-lib/flowy-core/src/user_state_callback.rs
rename to frontend/rust-lib/flowy-core/src/app_life_cycle.rs
index 50e0f980c1..0630bdf25c 100644
--- a/frontend/rust-lib/flowy-core/src/user_state_callback.rs
+++ b/frontend/rust-lib/flowy-core/src/app_life_cycle.rs
@@ -1,12 +1,14 @@
-use std::sync::{Arc, Weak};
-
use anyhow::Context;
use client_api::entity::billing_dto::SubscriptionPlan;
-use tracing::{error, event, info};
+use std::sync::{Arc, Weak};
+use tracing::{error, event, info, instrument};
+use crate::full_indexed_data_provider::FullIndexedDataProvider;
+use crate::indexed_data_consumer::{close_document_tantivy_state, get_document_tantivy_state};
use crate::server_layer::ServerProvider;
use collab_entity::CollabType;
use collab_integrate::collab_builder::AppFlowyCollabBuilder;
+use collab_integrate::instant_indexed_data_provider::InstantIndexedDataProvider;
use collab_plugins::local_storage::kv::doc::CollabKVAction;
use collab_plugins::local_storage::kv::KVTransactionDB;
use flowy_ai::ai_manager::AIManager;
@@ -14,16 +16,20 @@ use flowy_database2::DatabaseManager;
use flowy_document::manager::DocumentManager;
use flowy_error::{FlowyError, FlowyResult};
use flowy_folder::manager::{FolderInitDataSource, FolderManager};
+use flowy_search::services::manager::SearchManager;
+use flowy_server::af_cloud::define::LoggedUser;
use flowy_storage::manager::StorageManager;
-use flowy_user::event_map::UserStatusCallback;
+use flowy_user::event_map::AppLifeCycle;
+use flowy_user::services::entities::{UserConfig, UserPaths};
use flowy_user::user_manager::UserManager;
use flowy_user_pub::cloud::{UserCloudConfig, UserCloudServiceProvider};
use flowy_user_pub::entities::{UserProfile, UserWorkspace, WorkspaceType};
use lib_dispatch::runtime::AFPluginRuntime;
use lib_infra::async_trait::async_trait;
+use tokio::sync::RwLock;
use uuid::Uuid;
-pub(crate) struct UserStatusCallbackImpl {
+pub(crate) struct AppLifeCycleImpl {
pub(crate) user_manager: Weak<UserManager>,
pub(crate) collab_builder: Weak<AppFlowyCollabBuilder>,
pub(crate) folder_manager: Weak<FolderManager>,
@@ -31,56 +37,68 @@ pub(crate) struct UserStatusCallbackImpl {
pub(crate) document_manager: Weak<DocumentManager>,
pub(crate) server_provider: Weak<ServerProvider>,
pub(crate) storage_manager: Weak<StorageManager>,
+ pub(crate) search_manager: Weak<SearchManager>,
pub(crate) ai_manager: Weak<AIManager>,
+ pub(crate) instant_indexed_data_provider: Option<Arc<InstantIndexedDataProvider>>,
+ pub(crate) full_indexed_data_provider: Weak<RwLock<Option<FullIndexedDataProvider>>>,
+ pub(crate) logged_user: Arc<dyn LoggedUser>,
// By default, all callback will run on the caller thread. If you don't want to block the caller
// thread, you can use runtime to spawn a new task.
pub(crate) runtime: Arc<AFPluginRuntime>,
+ pub(crate) full_indexed_finish_sender: tokio::sync::watch::Sender<bool>,
}
-impl UserStatusCallbackImpl {
- fn user_manager(&self) -> Result<Arc<UserManager>, FlowyError> {
+impl AppLifeCycleImpl {
+ pub(crate) fn user_manager(&self) -> Result<Arc<UserManager>, FlowyError> {
self.user_manager.upgrade().ok_or_else(FlowyError::ref_drop)
}
- fn folder_manager(&self) -> Result<Arc<FolderManager>, FlowyError> {
+ pub(crate) fn folder_manager(&self) -> Result<Arc<FolderManager>, FlowyError> {
self
.folder_manager
.upgrade()
.ok_or_else(FlowyError::ref_drop)
}
- fn database_manager(&self) -> Result<Arc<DatabaseManager>, FlowyError> {
+ pub(crate) fn database_manager(&self) -> Result<Arc<DatabaseManager>, FlowyError> {
self
.database_manager
.upgrade()
.ok_or_else(FlowyError::ref_drop)
}
- fn document_manager(&self) -> Result<Arc<DocumentManager>, FlowyError> {
+ pub(crate) fn document_manager(&self) -> Result<Arc<DocumentManager>, FlowyError> {
self
.document_manager
.upgrade()
.ok_or_else(FlowyError::ref_drop)
}
- fn server_provider(&self) -> Result<Arc<ServerProvider>, FlowyError> {
+ pub(crate) fn server_provider(&self) -> Result<Arc<ServerProvider>, FlowyError> {
self
.server_provider
.upgrade()
.ok_or_else(FlowyError::ref_drop)
}
- fn storage_manager(&self) -> Result<Arc<StorageManager>, FlowyError> {
+ pub(crate) fn storage_manager(&self) -> Result<Arc<StorageManager>, FlowyError> {
self
.storage_manager
.upgrade()
.ok_or_else(FlowyError::ref_drop)
}
- fn ai_manager(&self) -> Result<Arc<AIManager>, FlowyError> {
+ pub(crate) fn ai_manager(&self) -> Result<Arc<AIManager>, FlowyError> {
self.ai_manager.upgrade().ok_or_else(FlowyError::ref_drop)
}
+ pub(crate) fn search_manager(&self) -> Result<Arc<SearchManager>, FlowyError> {
+ self
+ .search_manager
+ .upgrade()
+ .ok_or_else(FlowyError::ref_drop)
+ }
+
async fn folder_init_data_source(
&self,
user_id: i64,
@@ -119,13 +137,15 @@ impl UserStatusCallbackImpl {
}
#[async_trait]
-impl UserStatusCallback for UserStatusCallbackImpl {
+impl AppLifeCycle for AppLifeCycleImpl {
+ #[instrument(skip_all)]
async fn on_launch_if_authenticated(
&self,
user_id: i64,
cloud_config: &Option<UserCloudConfig>,
workspace_id: &Uuid,
- _device_id: &str,
+ user_config: &UserConfig,
+ user_paths: &UserPaths,
workspace_type: &WorkspaceType,
) -> FlowyResult<()> {
if let Some(cloud_config) = cloud_config {
@@ -156,8 +176,40 @@ impl UserStatusCallback for UserStatusCallbackImpl {
self.document_manager()?.initialize(user_id).await?;
let cloned_ai_manager = self.ai_manager()?;
+ let server_provider = self.server_provider()?;
+ self
+ .create_thanvity_state_if_not_exists(user_id, workspace_id, user_paths)
+ .await;
+ self
+ .start_full_indexed_data_provider(
+ user_id,
+ workspace_id,
+ workspace_type,
+ user_config,
+ user_paths,
+ )
+ .await;
+
+ self
+ .start_instant_indexed_data_provider(
+ user_id,
+ workspace_id,
+ workspace_type,
+ user_config,
+ user_paths,
+ )
+ .await;
+
+ // do not change the order of this function
+ self
+ .search_manager()?
+ .on_launch_if_authenticated(workspace_id, get_document_tantivy_state(workspace_id))
+ .await;
+
let workspace_id = *workspace_id;
+ let workspace_type = *workspace_type;
self.runtime.spawn(async move {
+ server_provider.on_launch_if_authenticated(&workspace_type);
if let Err(err) = cloned_ai_manager
.on_launch_if_authenticated(&workspace_id)
.await
@@ -165,22 +217,31 @@ impl UserStatusCallback for UserStatusCallbackImpl {
error!("Failed to initialize AIManager: {:?}", err);
}
});
+
Ok(())
}
+ #[instrument(skip_all)]
async fn on_sign_in(
&self,
user_id: i64,
workspace_id: &Uuid,
- device_id: &str,
+ user_config: &UserConfig,
+ user_paths: &UserPaths,
workspace_type: &WorkspaceType,
) -> FlowyResult<()> {
event!(
tracing::Level::TRACE,
"Notify did sign in: latest_workspace: {:?}, device_id: {}",
workspace_id,
- device_id
+ user_config.device_id,
);
+ let server_provider = self.server_provider()?;
+ let c_workspace_type = *workspace_type;
+ self.runtime.spawn(async move {
+ server_provider.on_sign_in(&c_workspace_type);
+ });
+
let data_source = self
.folder_init_data_source(user_id, workspace_id, workspace_type)
.await?;
@@ -202,15 +263,45 @@ impl UserStatusCallback for UserStatusCallbackImpl {
.initialize_after_sign_in(workspace_id)
.await?;
+ self
+ .create_thanvity_state_if_not_exists(user_id, workspace_id, user_paths)
+ .await;
+ self
+ .start_full_indexed_data_provider(
+ user_id,
+ workspace_id,
+ workspace_type,
+ user_config,
+ user_paths,
+ )
+ .await;
+
+ self
+ .start_instant_indexed_data_provider(
+ user_id,
+ workspace_id,
+ workspace_type,
+ user_config,
+ user_paths,
+ )
+ .await;
+
+ // do not change the order of this function
+ self
+ .search_manager()?
+ .initialize_after_sign_in(workspace_id, get_document_tantivy_state(workspace_id))
+ .await;
Ok(())
}
+ #[instrument(skip_all)]
async fn on_sign_up(
&self,
is_new_user: bool,
user_profile: &UserProfile,
workspace_id: &Uuid,
- device_id: &str,
+ user_config: &UserConfig,
+ user_paths: &UserPaths,
workspace_type: &WorkspaceType,
) -> FlowyResult<()> {
event!(
@@ -218,8 +309,14 @@ impl UserStatusCallback for UserStatusCallbackImpl {
"Notify did sign up: is new: {} user_workspace: {:?}, device_id: {}",
is_new_user,
workspace_id,
- device_id
+ user_config.device_id,
);
+ let c_workspace_type = *workspace_type;
+ let server_provider = self.server_provider()?;
+ self.runtime.spawn(async move {
+ server_provider.on_sign_in(&c_workspace_type);
+ });
+
let data_source = self
.folder_init_data_source(user_profile.uid, workspace_id, workspace_type)
.await?;
@@ -252,6 +349,34 @@ impl UserStatusCallback for UserStatusCallbackImpl {
.ai_manager()?
.initialize_after_sign_up(workspace_id)
.await?;
+
+ self
+ .create_thanvity_state_if_not_exists(user_profile.uid, workspace_id, user_paths)
+ .await;
+ self
+ .start_full_indexed_data_provider(
+ user_profile.uid,
+ workspace_id,
+ workspace_type,
+ user_config,
+ user_paths,
+ )
+ .await;
+ self
+ .start_instant_indexed_data_provider(
+ user_profile.uid,
+ workspace_id,
+ workspace_type,
+ user_config,
+ user_paths,
+ )
+ .await;
+
+ // do not change the order of this function
+ self
+ .search_manager()?
+ .initialize_after_sign_up(workspace_id, get_document_tantivy_state(workspace_id))
+ .await;
Ok(())
}
@@ -260,17 +385,31 @@ impl UserStatusCallback for UserStatusCallbackImpl {
Ok(())
}
+ async fn on_workspace_closed(&self, workspace_id: &Uuid) -> FlowyResult<()> {
+ close_document_tantivy_state(workspace_id);
+ Ok(())
+ }
+
+ #[instrument(skip_all)]
async fn on_workspace_opened(
&self,
user_id: i64,
workspace_id: &Uuid,
_user_workspace: &UserWorkspace,
workspace_type: &WorkspaceType,
+ user_config: &UserConfig,
+ user_paths: &UserPaths,
) -> FlowyResult<()> {
let data_source = self
.folder_init_data_source(user_id, workspace_id, workspace_type)
.await?;
+ let server_provider = self.server_provider()?;
+ let c_workspace_type = *workspace_type;
+ self.runtime.spawn(async move {
+ server_provider.init_after_open_workspace(&c_workspace_type);
+ });
+
self
.folder_manager()?
.initialize_after_open_workspace(user_id, data_source)
@@ -291,6 +430,42 @@ impl UserStatusCallback for UserStatusCallbackImpl {
.storage_manager()?
.initialize_after_open_workspace(workspace_id)
.await;
+
+ self
+ .create_thanvity_state_if_not_exists(user_id, workspace_id, user_paths)
+ .await;
+ self
+ .start_full_indexed_data_provider(
+ user_id,
+ workspace_id,
+ workspace_type,
+ user_config,
+ user_paths,
+ )
+ .await;
+ self
+ .start_instant_indexed_data_provider(
+ user_id,
+ workspace_id,
+ workspace_type,
+ user_config,
+ user_paths,
+ )
+ .await;
+
+ // do not change the order of this function
+ self
+ .search_manager()?
+ .initialize_after_open_workspace(workspace_id, get_document_tantivy_state(workspace_id))
+ .await;
+ Ok(())
+ }
+
+ async fn on_workspace_deleted(&self, user_id: i64, workspace_id: &Uuid) -> FlowyResult<()> {
+ self
+ .folder_manager()?
+ .on_workspace_deleted(user_id, workspace_id)
+ .await?;
Ok(())
}
@@ -329,6 +504,10 @@ impl UserStatusCallback for UserStatusCallbackImpl {
}
}
}
+
+ fn subscribe_full_indexed_finish(&self) -> Option<tokio::sync::watch::Receiver<bool>> {
+ Some(self.full_indexed_finish_sender.subscribe())
+ }
}
fn resolve_data_source(
diff --git a/frontend/rust-lib/flowy-core/src/config.rs b/frontend/rust-lib/flowy-core/src/config.rs
index 2bad578627..7436bfc2b6 100644
--- a/frontend/rust-lib/flowy-core/src/config.rs
+++ b/frontend/rust-lib/flowy-core/src/config.rs
@@ -30,6 +30,48 @@ pub struct AppFlowyCoreConfig {
pub cloud_config: Option<AFCloudConfiguration>,
}
impl AppFlowyCoreConfig {
+ pub fn new(
+ app_version: Version,
+ custom_application_path: String,
+ application_path: String,
+ device_id: String,
+ platform: String,
+ name: String,
+ ) -> Self {
+ let cloud_config = AFCloudConfiguration::from_env().ok();
+ // By default enable sync trace log
+ let log_crates = vec!["sync_trace_log".to_string()];
+ let storage_path = match &cloud_config {
+ None => custom_application_path,
+ Some(config) => make_user_data_folder(&custom_application_path, &config.base_url),
+ };
+
+ let log_filter = create_log_filter(
+ "info".to_owned(),
+ log_crates,
+ OperatingSystem::from(&platform),
+ );
+
+ AppFlowyCoreConfig {
+ app_version,
+ name,
+ storage_path,
+ application_path,
+ device_id,
+ platform,
+ log_filter,
+ cloud_config,
+ }
+ }
+
+ pub fn log_filter(mut self, level: &str, with_crates: Vec<String>) -> Self {
+ self.log_filter = create_log_filter(
+ level.to_owned(),
+ with_crates,
+ OperatingSystem::from(&self.platform),
+ );
+ self
+ }
pub fn ensure_path(&self) {
let create_if_needed = |path_str: &str, label: &str| {
let dir = std::path::Path::new(path_str);
@@ -121,48 +163,3 @@ fn make_user_data_folder(root: &str, url: &str) -> String {
.unwrap_or_else(|_| root.to_string())
}
}
-
-impl AppFlowyCoreConfig {
- pub fn new(
- app_version: Version,
- custom_application_path: String,
- application_path: String,
- device_id: String,
- platform: String,
- name: String,
- ) -> Self {
- let cloud_config = AFCloudConfiguration::from_env().ok();
- // By default enable sync trace log
- let log_crates = vec!["sync_trace_log".to_string()];
- let storage_path = match &cloud_config {
- None => custom_application_path,
- Some(config) => make_user_data_folder(&custom_application_path, &config.base_url),
- };
-
- let log_filter = create_log_filter(
- "info".to_owned(),
- log_crates,
- OperatingSystem::from(&platform),
- );
-
- AppFlowyCoreConfig {
- app_version,
- name,
- storage_path,
- application_path,
- device_id,
- platform,
- log_filter,
- cloud_config,
- }
- }
-
- pub fn log_filter(mut self, level: &str, with_crates: Vec<String>) -> Self {
- self.log_filter = create_log_filter(
- level.to_owned(),
- with_crates,
- OperatingSystem::from(&self.platform),
- );
- self
- }
-}
diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/chat_deps.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/chat_deps.rs
index a7d2bc15c1..9509f271a6 100644
--- a/frontend/rust-lib/flowy-core/src/deps_resolve/chat_deps.rs
+++ b/frontend/rust-lib/flowy-core/src/deps_resolve/chat_deps.rs
@@ -4,10 +4,10 @@ use collab::preclude::updates::decoder::Decode;
use collab::preclude::{Collab, StateVector};
use collab::util::is_change_since_sv;
use collab_entity::CollabType;
-use collab_integrate::persistence::collab_metadata_sql::AFCollabMetadata;
use flowy_ai::ai_manager::{AIExternalService, AIManager};
use flowy_ai::local_ai::controller::LocalAIController;
use flowy_ai_pub::cloud::ChatCloudService;
+use flowy_ai_pub::persistence::AFCollabMetadata;
use flowy_ai_pub::user_service::AIUserService;
use flowy_error::{FlowyError, FlowyResult};
use flowy_folder::ViewLayout;
diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/collab_deps.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/collab_deps.rs
index 078ee7359b..1596c5ae52 100644
--- a/frontend/rust-lib/flowy-core/src/deps_resolve/collab_deps.rs
+++ b/frontend/rust-lib/flowy-core/src/deps_resolve/collab_deps.rs
@@ -1,4 +1,5 @@
use collab_entity::CollabType;
+use collab_integrate::collab_builder::WorkspaceCollabIntegrate;
use collab_integrate::{CollabSnapshot, PersistenceError, SnapshotPersistence};
use diesel::dsl::count_star;
use diesel::SqliteConnection;
@@ -8,8 +9,6 @@ use flowy_sqlite::{
schema::{collab_snapshot, collab_snapshot::dsl},
};
use flowy_user::services::authenticate_user::AuthenticateUser;
-
-use collab_integrate::collab_builder::WorkspaceCollabIntegrate;
use lib_infra::util::timestamp;
use std::sync::{Arc, Weak};
use tracing::debug;
diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps/mod.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps/mod.rs
index c677110923..1be60c5a45 100644
--- a/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps/mod.rs
+++ b/frontend/rust-lib/flowy-core/src/deps_resolve/folder_deps/mod.rs
@@ -13,7 +13,6 @@ use flowy_error::{internal_error, FlowyError, FlowyResult};
use flowy_folder::entities::UpdateViewParams;
use flowy_folder::manager::{FolderManager, FolderUser};
use flowy_folder::ViewLayout;
-use flowy_search::folder::indexer::FolderIndexManagerImpl;
use flowy_sqlite::kv::KVStorePreferences;
use flowy_user::services::authenticate_user::AuthenticateUser;
use flowy_user::services::data_import::load_collab_by_object_id;
@@ -35,7 +34,6 @@ impl FolderDepsResolver {
authenticate_user: Weak<AuthenticateUser>,
collab_builder: Arc<AppFlowyCollabBuilder>,
server_provider: Weak<ServerProvider>,
- folder_indexer: Arc<FolderIndexManagerImpl>,
store_preferences: Arc<KVStorePreferences>,
) -> Arc<FolderManager> {
let user: Arc<dyn FolderUser> = Arc::new(FolderUserImpl {
@@ -47,7 +45,6 @@ impl FolderDepsResolver {
user.clone(),
collab_builder,
server_provider.clone(),
- folder_indexer,
store_preferences,
)
.unwrap(),
diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/search_deps.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/search_deps.rs
index b31853a803..3afa1db1bc 100644
--- a/frontend/rust-lib/flowy-core/src/deps_resolve/search_deps.rs
+++ b/frontend/rust-lib/flowy-core/src/deps_resolve/search_deps.rs
@@ -1,7 +1,5 @@
use flowy_folder::manager::FolderManager;
-use flowy_search::document::handler::DocumentSearchHandler;
-use flowy_search::folder::handler::FolderSearchHandler;
-use flowy_search::folder::indexer::FolderIndexManagerImpl;
+use flowy_search::document::cloud_search_handler::DocumentCloudSearchHandler;
use flowy_search::services::manager::SearchManager;
use flowy_search_pub::cloud::SearchCloudService;
use std::sync::Arc;
@@ -9,12 +7,14 @@ use std::sync::Arc;
pub struct SearchDepsResolver();
impl SearchDepsResolver {
pub async fn resolve(
- folder_indexer: Arc<FolderIndexManagerImpl>,
cloud_service: Arc<dyn SearchCloudService>,
folder_manager: Arc<FolderManager>,
) -> Arc<SearchManager> {
- let folder_handler = Arc::new(FolderSearchHandler::new(folder_indexer));
- let document_handler = Arc::new(DocumentSearchHandler::new(cloud_service, folder_manager));
- Arc::new(SearchManager::new(vec![folder_handler, document_handler]))
+ // let folder_handler = Arc::new(FolderSearchHandler::new(folder_indexer));
+ let document_handler = Arc::new(DocumentCloudSearchHandler::new(
+ cloud_service,
+ folder_manager,
+ ));
+ Arc::new(SearchManager::new(vec![document_handler]))
}
}
diff --git a/frontend/rust-lib/flowy-core/src/deps_resolve/user_deps.rs b/frontend/rust-lib/flowy-core/src/deps_resolve/user_deps.rs
index 482b46b76c..582f7da043 100644
--- a/frontend/rust-lib/flowy-core/src/deps_resolve/user_deps.rs
+++ b/frontend/rust-lib/flowy-core/src/deps_resolve/user_deps.rs
@@ -12,8 +12,6 @@ use flowy_user_pub::workspace_service::UserWorkspaceService;
use lib_infra::async_trait::async_trait;
use std::collections::HashMap;
use std::sync::{Arc, Weak};
-use tracing::info;
-use uuid::Uuid;
pub struct UserDepsResolver();
@@ -87,20 +85,4 @@ impl UserWorkspaceService for UserWorkspaceServiceImpl {
.await?;
Ok(())
}
-
- async fn did_delete_workspace(&self, workspace_id: &Uuid) -> FlowyResult<()> {
- // The remove_indices_for_workspace should not block the deletion of the workspace
- // Log the error and continue
- if let Err(err) = self
- .folder_manager
- .upgrade()
- .ok_or_else(FlowyError::ref_drop)?
- .remove_indices_for_workspace(workspace_id)
- .await
- {
- info!("Error removing indices for workspace: {}", err);
- }
-
- Ok(())
- }
}
diff --git a/frontend/rust-lib/flowy-core/src/folder_view_observer.rs b/frontend/rust-lib/flowy-core/src/folder_view_observer.rs
new file mode 100644
index 0000000000..093e5552d1
--- /dev/null
+++ b/frontend/rust-lib/flowy-core/src/folder_view_observer.rs
@@ -0,0 +1,62 @@
+use collab::core::collab::{IndexContent, IndexContentReceiver};
+use collab_folder::ViewIndexContent;
+use flowy_search::document::local_search_handler::DocumentTantivyState;
+use flowy_search_pub::entities::FolderViewObserver;
+use lib_infra::async_trait::async_trait;
+use std::sync::Weak;
+use tokio::sync::RwLock;
+use tracing::error;
+use uuid::Uuid;
+
+pub struct FolderViewObserverImpl {
+ state: Weak<RwLock<DocumentTantivyState>>,
+}
+
+impl FolderViewObserverImpl {
+ pub fn new(_workspace_id: &Uuid, state: Weak<RwLock<DocumentTantivyState>>) -> Self {
+ Self { state }
+ }
+}
+
+#[async_trait]
+impl FolderViewObserver for FolderViewObserverImpl {
+ async fn set_observer_rx(&self, mut rx: IndexContentReceiver) {
+ let state = self.state.clone();
+ tokio::spawn(async move {
+ while let Ok(msg) = rx.recv().await {
+ let state = match state.upgrade() {
+ Some(state) => state,
+ None => {
+ return;
+ },
+ };
+
+ match msg {
+ IndexContent::Create(value) => match serde_json::from_value::<ViewIndexContent>(value) {
+ Ok(view) => {
+ let _ = state.write().await.add_document_metadata(
+ &view.id,
+ view.name.clone(),
+ view.icon.clone(),
+ );
+ },
+ Err(err) => error!("FolderIndexManager error deserialize (create): {:?}", err),
+ },
+ IndexContent::Update(value) => match serde_json::from_value::<ViewIndexContent>(value) {
+ Ok(view) => {
+ let _ = state.write().await.add_document_metadata(
+ &view.id,
+ view.name.clone(),
+ view.icon.clone(),
+ );
+ },
+ Err(err) => error!("FolderIndexManager error deserialize (update): {:?}", err),
+ },
+ IndexContent::Delete(ids) => {
+ let _ = state.write().await.delete_documents(&ids);
+ },
+ }
+ }
+ });
+ }
+}
diff --git a/frontend/rust-lib/flowy-core/src/full_indexed_data_provider.rs b/frontend/rust-lib/flowy-core/src/full_indexed_data_provider.rs
new file mode 100644
index 0000000000..16de1faf4d
--- /dev/null
+++ b/frontend/rust-lib/flowy-core/src/full_indexed_data_provider.rs
@@ -0,0 +1,314 @@
+use client_api::entity::workspace_dto::ViewIcon;
+use collab::preclude::Collab;
+use collab_document::document::DocumentBody;
+use collab_entity::CollabType;
+use collab_folder::{View, ViewLayout};
+use collab_plugins::local_storage::kv::doc::CollabKVAction;
+use collab_plugins::local_storage::kv::KVTransactionDB;
+use flowy_ai_pub::entities::{UnindexedCollab, UnindexedCollabMetadata, UnindexedData};
+use flowy_ai_pub::persistence::{
+ batch_upsert_index_collab, select_indexed_collab_ids, IndexCollabRecordTable,
+};
+use flowy_error::{FlowyError, FlowyResult};
+use flowy_folder::manager::FolderManager;
+use flowy_server::af_cloud::define::LoggedUser;
+use flowy_server_pub::workspace_dto::IconType;
+use lib_infra::async_trait::async_trait;
+use std::collections::HashMap;
+use std::sync::{Arc, Weak};
+use tokio::sync::RwLock;
+use tokio_util::sync::CancellationToken;
+use tracing::{error, info, trace, warn};
+use uuid::Uuid;
+
+#[async_trait]
+pub trait FullIndexedDataConsumer: Send + Sync {
+ fn consumer_id(&self) -> String;
+ async fn consume_indexed_data(&self, uid: i64, data: &UnindexedCollab) -> FlowyResult<()>;
+}
+
+#[derive(Clone)]
+pub struct FullIndexedDataProvider {
+ workspace_id: Uuid,
+ folder_manager: Weak<FolderManager>,
+ logged_user: Weak<dyn LoggedUser>,
+ cancel_token: CancellationToken,
+ consumers: Arc<RwLock<Vec<Box<dyn FullIndexedDataConsumer>>>>,
+}
+
+impl FullIndexedDataProvider {
+ pub fn new(
+ workspace_id: Uuid,
+ folder_manager: Weak<FolderManager>,
+ logged_user: Weak<dyn LoggedUser>,
+ ) -> Self {
+ let cancel_token = CancellationToken::new();
+ let consumers = Arc::new(RwLock::new(Vec::new()));
+ Self {
+ workspace_id,
+ folder_manager,
+ cancel_token,
+ logged_user,
+ consumers,
+ }
+ }
+
+ pub async fn num_consumers(&self) -> usize {
+ let consumers = self.consumers.read().await;
+ consumers.len()
+ }
+
+ pub async fn register_full_indexed_consumer(&self, consumer: Box<dyn FullIndexedDataConsumer>) {
+ info!(
+ "[Indexing] Registering full index consumer: {}",
+ consumer.consumer_id()
+ );
+ let mut consumers = self.consumers.write().await;
+ consumers.push(consumer);
+ }
+
+ pub fn cancel_indexing(&self) {
+ self.cancel_token.cancel();
+ }
+
+ async fn is_workspace_changed(&self) -> bool {
+ if let Some(logged_user) = self.logged_user.upgrade() {
+ if let Ok(current_workspace_id) = logged_user.workspace_id() {
+ return current_workspace_id != self.workspace_id;
+ }
+ }
+ // If we can't determine, assume it changed to be safe
+ true
+ }
+
+ pub async fn full_index_unindexed_documents(&self) -> FlowyResult<()> {
+ if self.consumers.read().await.is_empty() {
+ warn!("[Indexing] Indexing cancelled: No consumers registered");
+ return Ok(());
+ }
+
+ let logged_user = self.logged_user.upgrade().ok_or_else(|| {
+ FlowyError::unauthorized()
+ .with_context("[Indexing] Failed to upgrade AuthenticateUser when indexing")
+ })?;
+
+ let uid = logged_user.user_id()?;
+ let workspace_id = logged_user.workspace_id()?;
+ let mut conn = logged_user.get_sqlite_db(uid)?;
+
+ let folder_manager = self
+ .folder_manager
+ .upgrade()
+ .ok_or_else(|| FlowyError::internal().with_context("Failed to upgrade FolderManager"))?;
+ let views = folder_manager.get_all_views().await?;
+ let view_ids = views.iter().map(|v| v.id.clone()).collect::<Vec<_>>();
+ let view_by_view_id = Arc::new(
+ views
+ .into_iter()
+ .map(|v| (v.id.clone(), v))
+ .collect::<HashMap<_, _>>(),
+ );
+ drop(folder_manager);
+
+ let indexed = select_indexed_collab_ids(&mut conn, workspace_id.to_string())?;
+ let unindex_ids = view_ids
+ .into_iter()
+ .filter(|id| !indexed.contains(id))
+ .collect::<Vec<_>>();
+
+ if unindex_ids.is_empty() {
+ info!("[Indexing] skip: No unindexed documents");
+ return Ok(());
+ }
+
+ // chunk the unindex_ids into smaller chunks
+ let chunk_size = 20;
+ info!(
+ "[Indexing] {} consumers start indexing {} unindexed documents",
+ self.consumers.read().await.len(),
+ unindex_ids.len()
+ );
+ let chunks = unindex_ids.chunks(chunk_size);
+ for chunk in chunks.into_iter() {
+ if self.is_workspace_changed().await {
+ info!("[Indexing] cancelled: Workspace changed");
+ break;
+ }
+
+ match self
+ .index_views(uid, &workspace_id, chunk.to_vec(), view_by_view_id.clone())
+ .await
+ {
+ Ok(unindexed) => {
+ if self.cancel_token.is_cancelled() {
+ info!("[Indexing] cancelled");
+ break;
+ }
+
+ for consumer in self.consumers.read().await.iter() {
+ for data in &unindexed {
+ trace!(
+ "[Indexing] {} consume {} unindexed data",
+ consumer.consumer_id(),
+ data.object_id
+ );
+ if let Err(err) = consumer.consume_indexed_data(uid, data).await {
+ error!(
+ "[Indexing] Failed to consume unindexed data: {}: {:?}",
+ consumer.consumer_id(),
+ err
+ );
+ }
+ }
+ }
+ if let Some(mut db) = self
+ .logged_user
+ .upgrade()
+ .and_then(|v| v.get_sqlite_db(uid).ok())
+ {
+ let rows = unindexed
+ .into_iter()
+ .map(|v| IndexCollabRecordTable {
+ oid: v.object_id.to_string(),
+ workspace_id: v.workspace_id.to_string(),
+ content_hash: v.data.content_hash(),
+ })
+ .collect::<Vec<_>>();
+
+ batch_upsert_index_collab(&mut db, rows)?;
+ }
+ },
+ Err(err) => {
+ error!("[Indexing] Failed to index documents: {:?}", err);
+ },
+ }
+
+ if self.cancel_token.is_cancelled() {
+ info!("[Indexing] Indexing cancelled");
+ break;
+ }
+
+ // Check if workspace has changed before sleep
+ if self.is_workspace_changed().await {
+ info!("[Indexing] Indexing cancelled: Workspace changed");
+ break;
+ }
+
+ tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ }
+ Ok(())
+ }
+
+ pub async fn index_views(
+ &self,
+ uid: i64,
+ workspace_id: &Uuid,
+ unindex_ids: Vec<String>,
+ view_by_id: Arc<HashMap<String, Arc<View>>>,
+ ) -> FlowyResult<Vec<UnindexedCollab>> {
+ // Early return for empty input
+ if unindex_ids.is_empty() {
+ return Ok(Vec::new());
+ }
+
+ let collab_db = self
+ .logged_user
+ .upgrade()
+ .and_then(|u| u.get_collab_db(uid).ok())
+ .and_then(|c| c.upgrade())
+ .ok_or_else(|| FlowyError::internal().with_context("Failed to upgrade CollabKVDB"))?;
+
+ // Filter out views that don't exist before the blocking task
+ let filtered_ids: Vec<_> = unindex_ids
+ .into_iter()
+ .filter(|id| view_by_id.contains_key(id))
+ .collect();
+
+ // Move everything needed into the blocking closure
+ let workspace_id = *workspace_id;
+ let handle = tokio::task::spawn_blocking(move || -> FlowyResult<Vec<UnindexedCollab>> {
+ let read_txn = collab_db.read_txn();
+ let mut results = Vec::with_capacity(filtered_ids.len());
+
+ for object_str in filtered_ids {
+ // We know the view exists because of the pre-filtering
+ let view = &view_by_id[&object_str];
+
+ // Skip Chat views immediately
+ let collab_type = match view.layout {
+ ViewLayout::Document => CollabType::Document,
+ ViewLayout::Grid | ViewLayout::Board | ViewLayout::Calendar => CollabType::Database,
+ ViewLayout::Chat => continue,
+ };
+
+ // Parse UUID once, outside the match
+ let object_id = match Uuid::parse_str(&object_str) {
+ Ok(id) => id,
+ Err(_) => continue, // Skip invalid UUIDs
+ };
+
+ // Create metadata once for reuse
+ let metadata = UnindexedCollabMetadata {
+ name: view.name.clone(),
+ icon: view.icon.as_ref().map(|icon| ViewIcon {
+ ty: IconType::from(icon.ty.clone() as u8),
+ value: icon.value.clone(),
+ }),
+ };
+
+ match collab_type {
+ CollabType::Document => {
+ // 1) Load into a Collab
+ let mut collab = Collab::new(uid, &object_str, "indexing_device", vec![], false);
+ let load_success = {
+ let mut txn = collab.transact_mut();
+ read_txn
+ .load_doc_with_txn(uid, &workspace_id.to_string(), &object_str, &mut txn)
+ .is_ok()
+ };
+
+ if load_success {
+ if let Some(document) = DocumentBody::from_collab(&collab) {
+ let paragraphs = document.paragraphs(collab.transact());
+ results.push(UnindexedCollab {
+ workspace_id,
+ object_id,
+ collab_type: CollabType::Document,
+ data: UnindexedData::Paragraphs(paragraphs),
+ metadata,
+ });
+ continue;
+ }
+ }
+ // When load fails or document extraction fails, use empty text
+ results.push(UnindexedCollab {
+ workspace_id,
+ object_id,
+ collab_type: CollabType::Document,
+ data: UnindexedData::Text(String::new()),
+ metadata,
+ });
+ },
+ CollabType::Database => {
+ results.push(UnindexedCollab {
+ workspace_id,
+ object_id,
+ collab_type: CollabType::Database,
+ data: UnindexedData::Text(String::new()),
+ metadata,
+ });
+ },
+ _ => {
+ // do nothing for other types
+ },
+ }
+ }
+
+ Ok(results)
+ });
+
+ handle
+ .await
+ .map_err(|e| FlowyError::internal().with_context(format!("Join error: {}", e)))?
+ }
+}
diff --git a/frontend/rust-lib/flowy-core/src/indexed_data_consumer.rs b/frontend/rust-lib/flowy-core/src/indexed_data_consumer.rs
new file mode 100644
index 0000000000..d387c6660b
--- /dev/null
+++ b/frontend/rust-lib/flowy-core/src/indexed_data_consumer.rs
@@ -0,0 +1,391 @@
+use crate::folder_view_observer::FolderViewObserverImpl;
+use crate::full_indexed_data_provider::FullIndexedDataConsumer;
+use collab_entity::{CollabObject, CollabType};
+use collab_folder::folder_diff::FolderViewChange;
+use collab_folder::{IconType, ViewIcon};
+use collab_integrate::instant_indexed_data_provider::InstantIndexedDataConsumer;
+use dashmap::DashMap;
+use flowy_ai_pub::entities::{UnindexedCollab, UnindexedCollabMetadata, UnindexedData};
+use flowy_error::{FlowyError, FlowyResult};
+use flowy_folder::manager::FolderManager;
+use flowy_search::document::local_search_handler::DocumentTantivyState;
+use flowy_search_pub::entities::FolderViewObserver;
+use flowy_server::af_cloud::define::LoggedUser;
+use lib_infra::async_trait::async_trait;
+use once_cell::sync::Lazy;
+use std::path::PathBuf;
+use std::sync::{Arc, Weak};
+use tokio::sync::RwLock;
+use tracing::{error, info, trace};
+use uuid::Uuid;
+
+#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
+pub struct EmbeddingFullIndexConsumer;
+
+#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
+#[async_trait]
+impl FullIndexedDataConsumer for EmbeddingFullIndexConsumer {
+ fn consumer_id(&self) -> String {
+ "embedding_full_index_consumer".to_string()
+ }
+
+ async fn consume_indexed_data(&self, _uid: i64, data: &UnindexedCollab) -> FlowyResult<()> {
+ if !matches!(data.collab_type, CollabType::Document) {
+ return Ok(());
+ }
+
+ if data.data.is_empty() {
+ return Ok(());
+ }
+
+ let scheduler = flowy_ai::embeddings::context::EmbedContext::shared().get_scheduler()?;
+ scheduler.index_collab(data.clone()).await?;
+ Ok(())
+ }
+}
+
+pub struct EmbeddingsInstantConsumerImpl {
+ consume_history: DashMap<String, String>,
+}
+
+impl EmbeddingsInstantConsumerImpl {
+ pub fn new() -> Self {
+ Self {
+ consume_history: Default::default(),
+ }
+ }
+}
+
+#[async_trait]
+impl InstantIndexedDataConsumer for EmbeddingsInstantConsumerImpl {
+ fn consumer_id(&self) -> String {
+ "embedding_instant_index_consumer".to_string()
+ }
+
+ async fn consume_collab(
+ &self,
+ collab_object: &CollabObject,
+ data: UnindexedData,
+ ) -> Result<bool, FlowyError> {
+ if data.is_empty() {
+ return Ok(false);
+ }
+
+ let content_hash = data.content_hash();
+ if let Some(entry) = self.consume_history.get(&collab_object.object_id) {
+ if entry.value() == &content_hash {
+ trace!(
+ "[Indexing] {} instant embedding already indexed, skipping",
+ collab_object.object_id
+ );
+ return Ok(false);
+ }
+ }
+
+ self
+ .consume_history
+ .insert(collab_object.object_id.clone(), content_hash);
+
+ #[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))]
+ {
+ use flowy_ai::embeddings::context::EmbedContext;
+ if let Ok(scheduler) = EmbedContext::shared().get_scheduler() {
+ let unindex_collab = UnindexedCollab {
+ workspace_id: Uuid::parse_str(&collab_object.workspace_id)?,
+ object_id: Uuid::parse_str(&collab_object.object_id)?,
+ collab_type: collab_object.collab_type,
+ data,
+ metadata: UnindexedCollabMetadata {
+ name: "".to_string(),
+ icon: None,
+ },
+ };
+
+ if let Err(err) = scheduler.index_collab(unindex_collab).await {
+ error!("[Embedding] error generating embedding: {}", err);
+ }
+ }
+ }
+
+ Ok(true)
+ }
+
+ async fn did_delete_collab(
+ &self,
+ workspace_id: &Uuid,
+ object_id: &Uuid,
+ ) -> Result<(), FlowyError> {
+ #[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))]
+ {
+ use flowy_ai::embeddings::context::EmbedContext;
+ if let Ok(scheduler) = EmbedContext::shared().get_scheduler() {
+ if let Err(err) = scheduler.delete_collab(workspace_id, object_id).await {
+ error!("[Embedding] error generating embedding: {}", err);
+ }
+ }
+ }
+
+ Ok(())
+ }
+}
+/// Global map: workspace_id → a *weak* handle to its index state.
+type DocIndexMap = DashMap<Uuid, Arc<RwLock<DocumentTantivyState>>>;
+static SEARCH_INDEX: Lazy<DocIndexMap> = Lazy::new(DocIndexMap::new);
+
+/// Returns a strong handle, creating it if needed.
+pub(crate) fn get_or_init_document_tantivy_state(
+ workspace_id: Uuid,
+ data_path: PathBuf,
+) -> FlowyResult<Arc<RwLock<DocumentTantivyState>>> {
+ Ok(
+ SEARCH_INDEX
+ .entry(workspace_id)
+ .or_try_insert_with(|| {
+ info!(
+ "[Indexing] Creating new tantivy state for workspace: {}",
+ workspace_id
+ );
+ let state = DocumentTantivyState::new(&workspace_id, data_path.clone())?;
+ let arc_state = Arc::new(RwLock::new(state));
+ Ok::<_, FlowyError>(arc_state)
+ })?
+ .clone(),
+ )
+}
+
+pub(crate) fn close_document_tantivy_state(workspace_id: &Uuid) {
+ if SEARCH_INDEX.remove(workspace_id).is_some() {
+ info!(
+ "[Indexing] close tantivy state for workspace: {}",
+ workspace_id
+ );
+ }
+}
+
+pub fn get_document_tantivy_state(
+ workspace_id: &Uuid,
+) -> Option<Weak<RwLock<DocumentTantivyState>>> {
+ if let Some(existing) = SEARCH_INDEX.get(workspace_id) {
+ return Some(Arc::downgrade(existing.value()));
+ }
+ None
+}
+/// -----------------------------------------------------
+/// Full‐index consumer holds only a Weak reference:
+/// -----------------------------------------------------
+pub struct SearchFullIndexConsumer {
+ workspace_id: Uuid,
+ state: Weak<RwLock<DocumentTantivyState>>,
+}
+
+impl SearchFullIndexConsumer {
+ pub fn new(workspace_id: &Uuid, data_path: PathBuf) -> FlowyResult<Self> {
+ let strong = get_or_init_document_tantivy_state(*workspace_id, data_path)?;
+ Ok(Self {
+ workspace_id: *workspace_id,
+ state: Arc::downgrade(&strong),
+ })
+ }
+}
+
+#[async_trait]
+impl FullIndexedDataConsumer for SearchFullIndexConsumer {
+ fn consumer_id(&self) -> String {
+ "search_full_index_consumer".into()
+ }
+
+ async fn consume_indexed_data(&self, _uid: i64, data: &UnindexedCollab) -> FlowyResult<()> {
+ if self.workspace_id != data.workspace_id {
+ return Ok(());
+ }
+
+ let strong = self
+ .state
+ .upgrade()
+ .ok_or_else(|| FlowyError::internal().with_context("Tantivy state dropped"))?;
+ let object_id = data.object_id.to_string();
+ let content = data.data.clone().into_string();
+
+ strong.write().await.add_document(
+ &object_id,
+ content,
+ data.metadata.name.clone(),
+ data.metadata.icon.clone().map(|v| ViewIcon {
+ ty: IconType::from(v.ty as u8),
+ value: v.value,
+ }),
+ )?;
+ Ok(())
+ }
+}
+
+/// -----------------------------------------------------
+/// Instant‐index consumer also holds a Weak:
+/// -----------------------------------------------------
+pub struct SearchInstantIndexImpl {
+ workspace_id: Uuid,
+ state: Weak<RwLock<DocumentTantivyState>>,
+ consume_history: DashMap<String, String>,
+ folder_manager: Weak<FolderManager>,
+ #[allow(dead_code)]
+ // bind the folder_observer lifetime to the SearchInstantIndexImpl
+ folder_observer: FolderViewObserverImpl,
+ logged_user: Weak<dyn LoggedUser>,
+}
+
+impl SearchInstantIndexImpl {
+ pub async fn new(
+ workspace_id: &Uuid,
+ data_path: PathBuf,
+ folder_manager: Weak<FolderManager>,
+ logged_user: Weak<dyn LoggedUser>,
+ ) -> FlowyResult<Self> {
+ let state = get_or_init_document_tantivy_state(*workspace_id, data_path)?;
+ let folder_observer = FolderViewObserverImpl::new(workspace_id, Arc::downgrade(&state));
+ if let Some(folder_manager) = folder_manager.upgrade() {
+ if let Ok(rx) = folder_manager.subscribe_folder_change_rx().await {
+ folder_observer.set_observer_rx(rx).await;
+ } else {
+ error!("[Indexing] Failed to subscribe to folder changes");
+ }
+ }
+
+ Ok(Self {
+ workspace_id: *workspace_id,
+ state: Arc::downgrade(&state),
+ consume_history: Default::default(),
+ folder_manager,
+ folder_observer,
+ logged_user,
+ })
+ }
+
+ pub fn refresh_search_index(&self) {
+ let weak_state = self.state.clone();
+ let folder_manager = self.folder_manager.clone();
+ let weak_logged_user = self.logged_user.clone();
+ let expected_workspace_id = self.workspace_id;
+
+ tokio::spawn(async move {
+ if weak_logged_user
+ .upgrade()
+ .and_then(|user| user.workspace_id().ok())
+ != Some(expected_workspace_id)
+ {
+ return Ok(());
+ }
+
+ if let (Some(folder_manager), Some(state)) = (folder_manager.upgrade(), weak_state.upgrade())
+ {
+ if let Ok(changes) = folder_manager.consumer_recent_workspace_changes().await {
+ let views = folder_manager.get_all_views().await?;
+ let views_map: std::collections::HashMap<String, _> = views
+ .into_iter()
+ .map(|view| (view.id.clone(), view))
+ .collect();
+
+ for change in changes {
+ match change {
+ FolderViewChange::Inserted { view_id } | FolderViewChange::Updated { view_id } => {
+ if let Some(view) = views_map.get(&view_id) {
+ let _ = state.write().await.add_document_metadata(
+ &view.id,
+ view.name.clone(),
+ view.icon.clone().map(|v| ViewIcon {
+ ty: IconType::from(v.ty as u8),
+ value: v.value,
+ }),
+ );
+ }
+ },
+ FolderViewChange::Deleted { view_ids } => {
+ let _ = state.write().await.delete_documents(
+ &view_ids
+ .iter()
+ .map(|id| id.to_string())
+ .collect::<Vec<String>>(),
+ );
+ },
+ }
+ }
+ }
+ }
+
+ Ok::<_, FlowyError>(())
+ });
+ }
+}
+
+#[async_trait]
+impl InstantIndexedDataConsumer for SearchInstantIndexImpl {
+ fn consumer_id(&self) -> String {
+ "search_instant_index_consumer".into()
+ }
+
+ async fn consume_collab(
+ &self,
+ collab_object: &CollabObject,
+ data: UnindexedData,
+ ) -> Result<bool, FlowyError> {
+ if self.workspace_id.to_string() != collab_object.workspace_id {
+ return Ok(false);
+ }
+
+ let state = self
+ .state
+ .upgrade()
+ .ok_or_else(|| FlowyError::internal().with_context("Tantivy state dropped"))?;
+
+ let folder_manager = self
+ .folder_manager
+ .upgrade()
+ .ok_or_else(FlowyError::ref_drop)?;
+ let view = folder_manager.get_view(&collab_object.object_id).await?;
+
+ // Create a combined hash that includes content + view name + icon
+ let content_hash = data.content_hash();
+ let name_hash = format!("{}:{}", content_hash, view.name);
+ let combined_hash = if let Some(icon) = &view.icon {
+ format!("{}:{}:{}", name_hash, icon.ty.clone() as u8, icon.value)
+ } else {
+ name_hash
+ };
+
+ if let Some(entry) = self.consume_history.get(&collab_object.object_id) {
+ if entry.value() == &combined_hash {
+ return Ok(false);
+ }
+ }
+
+ self
+ .consume_history
+ .insert(collab_object.object_id.clone(), combined_hash);
+
+ state.write().await.add_document(
+ &collab_object.object_id,
+ data.into_string(),
+ view.name.clone(),
+ view.icon.clone().map(|v| ViewIcon {
+ ty: IconType::from(v.ty as u8),
+ value: v.value,
+ }),
+ )?;
+ Ok(true)
+ }
+
+ async fn did_delete_collab(
+ &self,
+ _workspace_id: &Uuid,
+ object_id: &Uuid,
+ ) -> Result<(), FlowyError> {
+ let strong = self
+ .state
+ .upgrade()
+ .ok_or_else(|| FlowyError::internal().with_context("Tantivy state dropped"))?;
+ strong
+ .write()
+ .await
+ .delete_document(&object_id.to_string())?;
+ Ok(())
+ }
+}
diff --git a/frontend/rust-lib/flowy-core/src/indexing_data_runner.rs b/frontend/rust-lib/flowy-core/src/indexing_data_runner.rs
new file mode 100644
index 0000000000..5b33dc3be8
--- /dev/null
+++ b/frontend/rust-lib/flowy-core/src/indexing_data_runner.rs
@@ -0,0 +1,226 @@
+use crate::app_life_cycle::AppLifeCycleImpl;
+use crate::full_indexed_data_provider::FullIndexedDataProvider;
+use crate::indexed_data_consumer::{
+ get_or_init_document_tantivy_state, EmbeddingsInstantConsumerImpl, SearchFullIndexConsumer,
+ SearchInstantIndexImpl,
+};
+use flowy_folder::manager::FolderManager;
+use flowy_user::services::entities::{UserConfig, UserPaths};
+use flowy_user_pub::entities::WorkspaceType;
+use std::sync::{Arc, Weak};
+use std::time::Duration;
+use tokio::time::timeout;
+use tracing::{error, info, instrument, warn};
+use uuid::Uuid;
+
+impl AppLifeCycleImpl {
+ async fn wait_for_folder_ready(folder_manager: &Weak<FolderManager>) -> bool {
+ let mut folder_ready_notify = match folder_manager.upgrade() {
+ Some(folder) => folder.subscribe_folder_ready_notifier(),
+ None => {
+ warn!("[Indexing] No folder manager available, skipping indexed data provider");
+ return false;
+ },
+ };
+
+ // Check if the folder is already ready
+ if *folder_ready_notify.borrow() {
+ tokio::time::sleep(Duration::from_secs(5)).await;
+ return true;
+ }
+
+ // Wait for the folder to become ready with timeout
+ match timeout(Duration::from_secs(20), folder_ready_notify.changed()).await {
+ Err(_) => {
+ warn!("[Indexing] Timeout waiting for folder ready");
+ false
+ },
+ Ok(_) => {
+ let is_ready = *folder_ready_notify.borrow();
+ if is_ready {
+ // We don't want to start indexing immediately after the folder is ready
+ tokio::time::sleep(Duration::from_secs(5)).await;
+ }
+
+ is_ready
+ },
+ }
+ }
+
+ #[instrument(skip_all)]
+ pub(crate) async fn start_instant_indexed_data_provider(
+ &self,
+ user_id: i64,
+ workspace_id: &Uuid,
+ workspace_type: &WorkspaceType,
+ _user_config: &UserConfig,
+ user_paths: &UserPaths,
+ ) {
+ let instant_indexed_data_provider = self.instant_indexed_data_provider.clone();
+ let runtime = self.runtime.clone();
+ let workspace_id_cloned = *workspace_id;
+ let workspace_type_cloned = *workspace_type;
+ let user_paths = user_paths.clone();
+ let folder_manager = self.folder_manager.clone();
+ let weak_logged_user = Arc::downgrade(&self.logged_user);
+
+ self.runtime.spawn(async move {
+ if !Self::wait_for_folder_ready(&folder_manager).await {
+ return;
+ }
+
+ info!(
+ "[Indexing] Starting instant indexed data provider for workspace: {:?}",
+ workspace_id_cloned
+ );
+
+ if let Some(instant_indexed_data_provider) = instant_indexed_data_provider {
+ instant_indexed_data_provider.clear_consumers().await;
+
+ // Add embedding consumer when workspace type is local
+ #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
+ {
+ if workspace_type_cloned.is_local() {
+ instant_indexed_data_provider
+ .register_consumer(Box::new(EmbeddingsInstantConsumerImpl::new()))
+ .await;
+ }
+ }
+
+ match SearchInstantIndexImpl::new(
+ &workspace_id_cloned,
+ user_paths.tanvity_index_path(user_id),
+ folder_manager,
+ weak_logged_user,
+ )
+ .await
+ {
+ Ok(consumer) => {
+ consumer.refresh_search_index();
+ instant_indexed_data_provider
+ .register_consumer(Box::new(consumer))
+ .await;
+ },
+ Err(err) => error!(
+ "[Indexing] Failed to create SearchInstantIndexImpl: {:?}",
+ err
+ ),
+ }
+
+ if instant_indexed_data_provider.num_consumers().await > 0 {
+ info!(
+ "[Indexing] instant indexed data provider with {} consumers for workspace: {:?}",
+ instant_indexed_data_provider.num_consumers().await,
+ workspace_id_cloned
+ );
+ if let Err(err) = instant_indexed_data_provider
+ .spawn_instant_indexed_provider(&runtime.inner)
+ .await
+ {
+ error!(
+ "[Indexing] Failed to spawn instant indexed data provider: {:?}",
+ err
+ );
+ }
+ }
+ } else {
+ info!("[Indexing] No instant indexed data provider to start");
+ }
+ });
+ }
+
+ pub(crate) async fn create_thanvity_state_if_not_exists(
+ &self,
+ uid: i64,
+ workspace_id: &Uuid,
+ user_paths: &UserPaths,
+ ) {
+ let data_path = user_paths.tanvity_index_path(uid);
+ let _ = get_or_init_document_tantivy_state(*workspace_id, data_path);
+ }
+
+ #[instrument(skip(self, _user_config, user_paths))]
+ pub(crate) async fn start_full_indexed_data_provider(
+ &self,
+ uid: i64,
+ workspace_id: &Uuid,
+ workspace_type: &WorkspaceType,
+ _user_config: &UserConfig,
+ user_paths: &UserPaths,
+ ) {
+ let folder_manager = self.folder_manager.clone();
+ let logged_user = self.logged_user.clone();
+ let full_indexed_data_provider = self.full_indexed_data_provider.clone();
+ let workspace_id_cloned = *workspace_id;
+ let workspace_type_cloned = *workspace_type;
+ let user_paths = user_paths.clone();
+ let full_indexed_finish_sender = self.full_indexed_finish_sender.clone();
+ full_indexed_finish_sender.send_replace(false);
+
+ self.runtime.spawn(async move {
+ if !Self::wait_for_folder_ready(&folder_manager).await {
+ return;
+ }
+
+ info!(
+ "[Indexing] Starting full indexed data provider for workspace: {:?}",
+ workspace_id_cloned
+ );
+
+ let new_provider = FullIndexedDataProvider::new(
+ workspace_id_cloned,
+ folder_manager,
+ Arc::downgrade(&logged_user),
+ );
+ #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
+ {
+ if workspace_type_cloned.is_local() {
+ new_provider
+ .register_full_indexed_consumer(Box::new(
+ crate::indexed_data_consumer::EmbeddingFullIndexConsumer,
+ ))
+ .await;
+ }
+ }
+
+ match SearchFullIndexConsumer::new(&workspace_id_cloned, user_paths.tanvity_index_path(uid)) {
+ Ok(consumer) => {
+ new_provider
+ .register_full_indexed_consumer(Box::new(consumer))
+ .await;
+ },
+ Err(err) => error!(
+ "[Indexing] Failed to create SearchFullIndexConsumer: {:?}",
+ err
+ ),
+ }
+
+ if new_provider.num_consumers().await > 0 {
+ info!(
+ "[Indexing] full indexed data provider with {} consumers for workspace: {:?}",
+ new_provider.num_consumers().await,
+ workspace_id_cloned
+ );
+ match new_provider.full_index_unindexed_documents().await {
+ Ok(()) => {
+ info!("[Indexing] full index succeeded");
+ },
+ Err(err) => {
+ error!("[Indexing] full index failed {:?}", err);
+ },
+ }
+ full_indexed_finish_sender.send_replace(true);
+ info!("[Indexing] full indexed data provider stopped");
+ }
+
+ if let Some(provider) = full_indexed_data_provider.upgrade() {
+ let old = provider.write().await.replace(new_provider);
+ if let Some(old) = old {
+ old.cancel_indexing();
+ }
+ } else {
+ info!("[Indexing] No full indexed data provider to start");
+ }
+ });
+ }
+}
diff --git a/frontend/rust-lib/flowy-core/src/lib.rs b/frontend/rust-lib/flowy-core/src/lib.rs
index fe7e69649c..19c5868a45 100644
--- a/frontend/rust-lib/flowy-core/src/lib.rs
+++ b/frontend/rust-lib/flowy-core/src/lib.rs
@@ -1,15 +1,20 @@
#![allow(unused_doc_comments)]
use collab_integrate::collab_builder::AppFlowyCollabBuilder;
+use collab_integrate::instant_indexed_data_provider::InstantIndexedDataProvider;
use collab_plugins::CollabKVDB;
use flowy_ai::ai_manager::AIManager;
use flowy_database2::DatabaseManager;
use flowy_document::manager::DocumentManager;
use flowy_error::{FlowyError, FlowyResult};
use flowy_folder::manager::FolderManager;
-use flowy_search::folder::indexer::FolderIndexManagerImpl;
use flowy_search::services::manager::SearchManager;
use flowy_server::af_cloud::define::LoggedUser;
+use flowy_sqlite::kv::KVStorePreferences;
+use flowy_storage::manager::StorageManager;
+use flowy_user::services::authenticate_user::AuthenticateUser;
+use flowy_user::services::entities::UserConfig;
+use flowy_user::user_manager::UserManager;
use std::path::PathBuf;
use std::sync::{Arc, Weak};
use std::time::Duration;
@@ -18,35 +23,34 @@ use tokio::sync::RwLock;
use tracing::{debug, error, event, info, instrument};
use uuid::Uuid;
-use flowy_sqlite::kv::KVStorePreferences;
-use flowy_storage::manager::StorageManager;
-use flowy_user::services::authenticate_user::AuthenticateUser;
-use flowy_user::services::entities::UserConfig;
-use flowy_user::user_manager::UserManager;
-
use lib_dispatch::prelude::*;
use lib_dispatch::runtime::AFPluginRuntime;
use lib_infra::priority_task::{TaskDispatcher, TaskRunner};
-use lib_infra::util::OperatingSystem;
+use lib_infra::util::{get_operating_system, OperatingSystem};
use lib_log::stream_log::StreamLogSender;
use module::make_plugins;
use crate::config::AppFlowyCoreConfig;
use crate::deps_resolve::file_storage_deps::FileStorageResolver;
use crate::deps_resolve::*;
+use crate::full_indexed_data_provider::FullIndexedDataProvider;
use crate::log_filter::init_log;
use crate::server_layer::ServerProvider;
+use app_life_cycle::AppLifeCycleImpl;
use deps_resolve::reminder_deps::CollabInteractImpl;
use flowy_sqlite::DBConnection;
use lib_infra::async_trait::async_trait;
-use user_state_callback::UserStatusCallbackImpl;
+pub(crate) mod app_life_cycle;
pub mod config;
mod deps_resolve;
+mod folder_view_observer;
+mod full_indexed_data_provider;
+mod indexed_data_consumer;
+mod indexing_data_runner;
mod log_filter;
pub mod module;
pub(crate) mod server_layer;
-pub(crate) mod user_state_callback;
/// This name will be used as to identify the current [AppFlowyCore] instance.
/// Don't change this.
@@ -68,6 +72,7 @@ pub struct AppFlowyCore {
pub ai_manager: Arc<AIManager>,
pub storage_manager: Arc<StorageManager>,
pub collab_builder: Arc<AppFlowyCollabBuilder>,
+ pub indexed_data_provider: Arc<RwLock<Option<FullIndexedDataProvider>>>,
}
impl Drop for AppFlowyCore {
@@ -122,6 +127,10 @@ impl AppFlowyCore {
let store_preference = Arc::new(KVStorePreferences::new(&config.storage_path).unwrap());
info!("🔥{:?}", &config);
+ #[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))]
+ flowy_ai::embeddings::context::EmbedContext::shared()
+ .init_vector_db(PathBuf::from(&config.storage_path));
+
let task_scheduler = TaskDispatcher::new(Duration::from_secs(10));
let task_dispatcher = Arc::new(RwLock::new(task_scheduler));
runtime.spawn(TaskRunner::run(task_dispatcher.clone()));
@@ -158,31 +167,34 @@ impl AppFlowyCore {
search_manager,
ai_manager,
storage_manager,
+ instant_indexed_data_provider,
) = async {
let storage_manager = FileStorageResolver::resolve(
Arc::downgrade(&authenticate_user),
server_provider.clone(),
&user_config.storage_path,
);
+
+ let instant_indexed_data_provider = if get_operating_system().is_desktop() {
+ Some(Arc::new(InstantIndexedDataProvider::new()))
+ } else {
+ None
+ };
/// The shared collab builder is used to build the [Collab] instance. The plugins will be loaded
/// on demand based on the [CollabPluginConfig].
let collab_builder = Arc::new(AppFlowyCollabBuilder::new(
server_provider.clone(),
WorkspaceCollabIntegrateImpl(Arc::downgrade(&authenticate_user)),
+ instant_indexed_data_provider.as_ref().map(Arc::downgrade),
));
collab_builder
.set_snapshot_persistence(Arc::new(SnapshotDBImpl(Arc::downgrade(&authenticate_user))));
- let folder_indexer = Arc::new(FolderIndexManagerImpl::new(Arc::downgrade(
- &authenticate_user,
- )));
-
let folder_manager = FolderDepsResolver::resolve(
Arc::downgrade(&authenticate_user),
collab_builder.clone(),
Arc::downgrade(&server_provider),
- folder_indexer.clone(),
store_preference.clone(),
)
.await;
@@ -229,12 +241,8 @@ impl AppFlowyCore {
)
.await;
- let search_manager = SearchDepsResolver::resolve(
- folder_indexer,
- server_provider.clone(),
- folder_manager.clone(),
- )
- .await;
+ let search_manager =
+ SearchDepsResolver::resolve(server_provider.clone(), folder_manager.clone()).await;
// Register the folder operation handlers
register_handlers(
@@ -254,11 +262,14 @@ impl AppFlowyCore {
search_manager,
ai_manager,
storage_manager,
+ instant_indexed_data_provider,
)
}
.await;
- let user_status_callback = UserStatusCallbackImpl {
+ let indexed_data_provider = Arc::new(RwLock::new(None));
+ let (full_indexed_finish_sender, _) = tokio::sync::watch::channel(false);
+ let app_life_cycle = AppLifeCycleImpl {
user_manager: Arc::downgrade(&user_manager),
collab_builder: Arc::downgrade(&collab_builder),
folder_manager: Arc::downgrade(&folder_manager),
@@ -267,7 +278,12 @@ impl AppFlowyCore {
server_provider: Arc::downgrade(&server_provider),
storage_manager: Arc::downgrade(&storage_manager),
ai_manager: Arc::downgrade(&ai_manager),
+ search_manager: Arc::downgrade(&search_manager),
+ instant_indexed_data_provider,
+ full_indexed_data_provider: Arc::downgrade(&indexed_data_provider),
+ logged_user: Arc::new(ServerUserImpl(Arc::downgrade(&authenticate_user))),
runtime: runtime.clone(),
+ full_indexed_finish_sender,
};
let collab_interact_impl = CollabInteractImpl {
@@ -275,7 +291,7 @@ impl AppFlowyCore {
document_manager: Arc::downgrade(&document_manager),
};
if let Err(err) = user_manager
- .init_with_callback(user_status_callback, collab_interact_impl)
+ .init_with_callback(app_life_cycle, collab_interact_impl)
.await
{
error!("Init user failed: {}", err)
@@ -309,6 +325,7 @@ impl AppFlowyCore {
ai_manager,
storage_manager,
collab_builder,
+ indexed_data_provider,
}
}
diff --git a/frontend/rust-lib/flowy-core/src/server_layer.rs b/frontend/rust-lib/flowy-core/src/server_layer.rs
index 6e5d35d726..5b8b3d5585 100644
--- a/frontend/rust-lib/flowy-core/src/server_layer.rs
+++ b/frontend/rust-lib/flowy-core/src/server_layer.rs
@@ -31,6 +31,7 @@ pub struct ServerProvider {
// Our little guard wrapper:
pub struct ServerHandle<'a>(Ref<'a, AuthType, Arc<dyn AppFlowyServer>>);
+#[allow(clippy::needless_lifetimes)]
impl<'a> Deref for ServerHandle<'a> {
type Target = dyn AppFlowyServer;
fn deref(&self) -> &Self::Target {
@@ -78,6 +79,23 @@ impl ServerProvider {
}
}
+ pub fn on_launch_if_authenticated(&self, _workspace_type: &WorkspaceType) {
+ self.local_ai.reload_ollama_client();
+ }
+
+ pub fn on_sign_in(&self, _workspace_type: &WorkspaceType) {
+ self.local_ai.reload_ollama_client();
+ }
+
+ pub fn on_sign_up(&self, workspace_type: &WorkspaceType) {
+ if workspace_type.is_local() {
+ self.local_ai.reload_ollama_client();
+ }
+ }
+ pub fn init_after_open_workspace(&self, _workspace_type: &WorkspaceType) {
+ self.local_ai.reload_ollama_client();
+ }
+
pub fn set_auth_type(&self, new_auth_type: AuthType) {
let old_type = self.get_auth_type();
if old_type != new_auth_type {
diff --git a/frontend/rust-lib/flowy-database2/src/services/database_view/view_editor.rs b/frontend/rust-lib/flowy-database2/src/services/database_view/view_editor.rs
index 8b9b7032c7..e287373142 100644
--- a/frontend/rust-lib/flowy-database2/src/services/database_view/view_editor.rs
+++ b/frontend/rust-lib/flowy-database2/src/services/database_view/view_editor.rs
@@ -387,11 +387,11 @@ impl DatabaseViewEditor {
let _ = self.filter_controller.filter_rows_and_notify(rows).await;
}
- pub async fn v_sort_rows(&self, rows: &mut Vec<Arc<Row>>) {
+ pub async fn v_sort_rows(&self, rows: &mut [Arc<Row>]) {
self.sort_controller.write().await.sort_rows(rows).await
}
- pub async fn v_sort_rows_and_notify(&self, rows: &mut Vec<Arc<Row>>) {
+ pub async fn v_sort_rows_and_notify(&self, rows: &mut [Arc<Row>]) {
self
.sort_controller
.write()
diff --git a/frontend/rust-lib/flowy-database2/src/services/filter/entities.rs b/frontend/rust-lib/flowy-database2/src/services/filter/entities.rs
index 52e368596c..1d9b8990aa 100644
--- a/frontend/rust-lib/flowy-database2/src/services/filter/entities.rs
+++ b/frontend/rust-lib/flowy-database2/src/services/filter/entities.rs
@@ -35,10 +35,9 @@ impl Filter {
/// multiple AND/OR filters but no Data filters is considered "empty".
pub fn is_empty(&self) -> bool {
match &self.inner {
- FilterInner::And { children } | FilterInner::Or { children } => children
- .iter()
- .map(|filter| filter.is_empty())
- .all(|is_empty| is_empty),
+ FilterInner::And { children } | FilterInner::Or { children } => {
+ children.iter().all(|filter| filter.is_empty())
+ },
FilterInner::Data { .. } => false,
}
}
diff --git a/frontend/rust-lib/flowy-database2/src/services/group/entities.rs b/frontend/rust-lib/flowy-database2/src/services/group/entities.rs
index 296a9960f7..4c3ddc4017 100644
--- a/frontend/rust-lib/flowy-database2/src/services/group/entities.rs
+++ b/frontend/rust-lib/flowy-database2/src/services/group/entities.rs
@@ -147,11 +147,8 @@ impl GroupData {
pub fn add_row(&mut self, row: Row) {
#[cfg(feature = "verbose_log")]
tracing::trace!("[Database Group]: Add row:{} to group:{}", row.id, self.id);
- match self.rows.iter().find(|r| r.id == row.id) {
- None => {
- self.rows.push(row);
- },
- Some(_) => {},
+ if !self.rows.iter().any(|r| r.id == row.id) {
+ self.rows.push(row);
}
}
diff --git a/frontend/rust-lib/flowy-database2/src/services/sort/controller.rs b/frontend/rust-lib/flowy-database2/src/services/sort/controller.rs
index 8dc3393a0f..5df4629839 100644
--- a/frontend/rust-lib/flowy-database2/src/services/sort/controller.rs
+++ b/frontend/rust-lib/flowy-database2/src/services/sort/controller.rs
@@ -183,7 +183,7 @@ impl SortController {
self.task_scheduler.write().await.add_task(task);
}
- pub async fn sort_rows_and_notify(&mut self, rows: &mut Vec<Arc<Row>>) {
+ pub async fn sort_rows_and_notify(&mut self, rows: &mut [Arc<Row>]) {
self.sort_rows(rows).await;
let row_orders = rows
.iter()
@@ -202,7 +202,7 @@ impl SortController {
));
}
- pub async fn sort_rows(&mut self, rows: &mut Vec<Arc<Row>>) {
+ pub async fn sort_rows(&mut self, rows: &mut [Arc<Row>]) {
let fields = self.delegate.get_fields(&self.view_id, None).await;
for sort in self.sorts.iter().rev() {
rows.par_sort_by(|left, right| cmp_row(left, right, sort, &fields, &self.cell_cache));
diff --git a/frontend/rust-lib/flowy-database2/tests/database/filter_test/script.rs b/frontend/rust-lib/flowy-database2/tests/database/filter_test/script.rs
index 6592c04305..ec3c73e0e0 100644
--- a/frontend/rust-lib/flowy-database2/tests/database/filter_test/script.rs
+++ b/frontend/rust-lib/flowy-database2/tests/database/filter_test/script.rs
@@ -56,29 +56,39 @@ impl DatabaseFilterTest {
fn find_filter(
filter: &FilterPB,
- filter_type: FilterType,
- field_type: Option<FieldType>,
+ target_type: FilterType,
+ target_field: Option<FieldType>,
) -> Option<FilterPB> {
- match &filter.filter_type {
- FilterType::And | FilterType::Or if filter.filter_type == filter_type => Some(filter.clone()),
- FilterType::And | FilterType::Or => {
- for child_filter in filter.children.iter() {
- if let Some(result) = Self::find_filter(child_filter, filter_type, field_type) {
- return Some(result);
+ // 1) Does this node match the requested filter type?
+ if filter.filter_type == target_type {
+ match &filter.filter_type {
+ // Logical nodes match immediately
+ FilterType::And | FilterType::Or => {
+ return Some(filter.clone());
+ },
+
+ // Data nodes match only if their field_type matches the target
+ FilterType::Data => {
+ if let (Some(data), Some(field)) = (&filter.data, target_field) {
+ if data.field_type == field {
+ return Some(filter.clone());
+ }
}
+ },
+ }
+ }
+
+ // 2) If it’s a logical node, dive into its children
+ if matches!(filter.filter_type, FilterType::And | FilterType::Or) {
+ for child in &filter.children {
+ if let Some(found) = Self::find_filter(child, target_type, target_field) {
+ return Some(found);
}
- None
- },
- FilterType::Data
- if filter.filter_type == filter_type
- && field_type.map_or(false, |field_type| {
- field_type == filter.data.clone().unwrap().field_type
- }) =>
- {
- Some(filter.clone())
- },
- _ => None,
+ }
}
+
+ // 3) No match here nor below
+ None
}
pub async fn update_text_cell_with_change(
diff --git a/frontend/rust-lib/flowy-document/Cargo.toml b/frontend/rust-lib/flowy-document/Cargo.toml
index aaaef4938e..23547c64fe 100644
--- a/frontend/rust-lib/flowy-document/Cargo.toml
+++ b/frontend/rust-lib/flowy-document/Cargo.toml
@@ -29,11 +29,9 @@ serde.workspace = true
serde_json.workspace = true
tracing.workspace = true
tokio = { workspace = true, features = ["rt"] }
-anyhow.workspace = true
indexmap = { version = "2.1.0", features = ["serde"] }
uuid.workspace = true
futures.workspace = true
-tokio-stream = { workspace = true, features = ["sync"] }
dashmap.workspace = true
scraper = "0.18.0"
diff --git a/frontend/rust-lib/flowy-document/src/manager.rs b/frontend/rust-lib/flowy-document/src/manager.rs
index bfb9f885ce..445911d4b6 100644
--- a/frontend/rust-lib/flowy-document/src/manager.rs
+++ b/frontend/rust-lib/flowy-document/src/manager.rs
@@ -63,7 +63,7 @@ pub struct DocumentManager {
impl Drop for DocumentManager {
fn drop(&mut self) {
- tracing::trace!("[Drop] drop document manager");
+ trace!("[Drop] drop document manager");
}
}
diff --git a/frontend/rust-lib/flowy-document/tests/document/util.rs b/frontend/rust-lib/flowy-document/tests/document/util.rs
index 7315bd1753..ef92f32b59 100644
--- a/frontend/rust-lib/flowy-document/tests/document/util.rs
+++ b/frontend/rust-lib/flowy-document/tests/document/util.rs
@@ -42,6 +42,7 @@ impl DocumentTest {
WorkspaceCollabIntegrateImpl {
workspace_id: user.workspace_id,
},
+ None,
));
let manager = DocumentManager::new(
diff --git a/frontend/rust-lib/flowy-error/src/code.rs b/frontend/rust-lib/flowy-error/src/code.rs
index de6faa3c4d..da978218df 100644
--- a/frontend/rust-lib/flowy-error/src/code.rs
+++ b/frontend/rust-lib/flowy-error/src/code.rs
@@ -387,17 +387,20 @@ pub enum ErrorCode {
#[error("Reference resource is not available")]
WeakRefDrop = 132,
+ #[error("Local embedding not ready")]
+ LocalEmbeddingNotReady = 133,
+
#[error("New password is too weak")]
- NewPasswordTooWeak = 133,
+ NewPasswordTooWeak = 134,
#[error("Invalid new password")]
- InvalidNewPassword = 134,
+ InvalidNewPassword = 135,
#[error("New password is too long")]
- NewPasswordTooLong = 135,
+ NewPasswordTooLong = 136,
#[error("New password is too short")]
- NewPasswordTooShort = 136,
+ NewPasswordTooShort = 137,
}
impl ErrorCode {
diff --git a/frontend/rust-lib/flowy-folder/src/manager.rs b/frontend/rust-lib/flowy-folder/src/manager.rs
index e5809cea05..50f0c7b4c9 100644
--- a/frontend/rust-lib/flowy-folder/src/manager.rs
+++ b/frontend/rust-lib/flowy-folder/src/manager.rs
@@ -19,9 +19,10 @@ use crate::view_operation::{
use arc_swap::ArcSwapOption;
use client_api::entity::workspace_dto::PublishInfoView;
use client_api::entity::PublishInfo;
-use collab::core::collab::DataSource;
+use collab::core::collab::{DataSource, IndexContentReceiver};
use collab::lock::RwLock;
use collab_entity::{CollabType, EncodedCollab};
+use collab_folder::folder_diff::FolderViewChange;
use collab_folder::hierarchy_builder::{ParentChildViews, ViewExtraBuilder};
use collab_folder::{
Folder, FolderData, FolderNotify, Section, SectionItem, TrashInfo, View, ViewLayout, ViewUpdate,
@@ -37,7 +38,6 @@ use flowy_folder_pub::entities::{
PublishDatabaseData, PublishDatabasePayload, PublishDocumentPayload, PublishPayload,
PublishViewInfo, PublishViewMeta, PublishViewMetaData,
};
-use flowy_search_pub::entities::FolderIndexManager;
use flowy_sqlite::kv::KVStorePreferences;
use futures::future;
use std::collections::HashMap;
@@ -62,8 +62,8 @@ pub struct FolderManager {
pub(crate) user: Arc<dyn FolderUser>,
pub(crate) operation_handlers: FolderOperationHandlers,
pub cloud_service: Weak<dyn FolderCloudService>,
- pub(crate) folder_indexer: Arc<dyn FolderIndexManager>,
pub(crate) store_preferences: Arc<KVStorePreferences>,
+ pub(crate) folder_ready_notifier: tokio::sync::watch::Sender<bool>,
}
impl Drop for FolderManager {
@@ -77,22 +77,26 @@ impl FolderManager {
user: Arc<dyn FolderUser>,
collab_builder: Arc<AppFlowyCollabBuilder>,
cloud_service: Weak<dyn FolderCloudService>,
- folder_indexer: Arc<dyn FolderIndexManager>,
store_preferences: Arc<KVStorePreferences>,
) -> FlowyResult<Self> {
+ let (folder_ready_notifier, _) = tokio::sync::watch::channel(false);
let manager = Self {
user,
mutex_folder: Default::default(),
collab_builder,
operation_handlers: Default::default(),
cloud_service,
- folder_indexer,
store_preferences,
+ folder_ready_notifier,
};
Ok(manager)
}
+ pub fn subscribe_folder_ready_notifier(&self) -> tokio::sync::watch::Receiver<bool> {
+ self.folder_ready_notifier.subscribe()
+ }
+
pub fn cloud_service(&self) -> FlowyResult<Arc<dyn FolderCloudService>> {
self
.cloud_service
@@ -309,6 +313,43 @@ impl FolderManager {
self.initialize_after_sign_in(uid, data_source).await
}
+ pub async fn subscribe_folder_change_rx(&self) -> FlowyResult<IndexContentReceiver> {
+ let folder = self
+ .mutex_folder
+ .load_full()
+ .ok_or_else(folder_not_init_error)?;
+ let read_guard = folder.read().await;
+ Ok(read_guard.subscribe_index_content())
+ }
+
+ pub async fn consumer_recent_workspace_changes(&self) -> FlowyResult<Vec<FolderViewChange>> {
+ let folder = self
+ .mutex_folder
+ .load_full()
+ .ok_or_else(folder_not_init_error)?;
+ let workspace_id = self.user.workspace_id()?.to_string();
+ let encoded_collab = self
+ .store_preferences
+ .get_object::<EncodedCollab>(&workspace_id);
+
+ if encoded_collab.is_none() {
+ return Ok(vec![]);
+ }
+
+ let folder = folder.read().await;
+ let changes = folder.calculate_view_changes(encoded_collab.unwrap())?;
+
+ let encoded_collab = folder.encode_collab();
+ if let Ok(encoded) = encoded_collab {
+ let _ = self.store_preferences.set_object(&workspace_id, &encoded);
+ }
+ Ok(changes)
+ }
+
+ pub async fn on_workspace_deleted(&self, _uid: i64, _workspace_id: &Uuid) -> FlowyResult<()> {
+ Ok(())
+ }
+
/// Initialize the folder for the new user.
/// Using the [DefaultFolderBuilder] to create the default workspace for the new user.
#[instrument(level = "info", skip_all, err)]
@@ -1659,6 +1700,20 @@ impl FolderManager {
self.get_sections(Section::Favorite).await
}
+ pub async fn get_all_views(&self) -> FlowyResult<Vec<Arc<View>>> {
+ let lock = self
+ .mutex_folder
+ .load_full()
+ .ok_or_else(folder_not_init_error)?;
+ let views = lock
+ .read()
+ .await
+ .get_all_views()
+ .into_iter()
+ .collect::<Vec<_>>();
+ Ok(views)
+ }
+
#[tracing::instrument(level = "debug", skip(self))]
pub(crate) async fn get_my_recent_sections(&self) -> Vec<SectionItem> {
self.get_sections(Section::Recent).await
@@ -2039,15 +2094,6 @@ impl FolderManager {
.filter(|id| !my_private_view_ids.contains(id))
.collect()
}
-
- pub async fn remove_indices_for_workspace(&self, workspace_id: &Uuid) -> FlowyResult<()> {
- self
- .folder_indexer
- .remove_indices_for_workspace(*workspace_id)
- .await?;
-
- Ok(())
- }
}
/// Return the views that belong to the workspace. The views are filtered by the trash and all the private views.
diff --git a/frontend/rust-lib/flowy-folder/src/manager_init.rs b/frontend/rust-lib/flowy-folder/src/manager_init.rs
index 25f0b4989e..69dcb0bc25 100644
--- a/frontend/rust-lib/flowy-folder/src/manager_init.rs
+++ b/frontend/rust-lib/flowy-folder/src/manager_init.rs
@@ -3,13 +3,12 @@ use crate::manager_observer::*;
use crate::user_default::DefaultFolderBuilder;
use collab::core::collab::DataSource;
use collab::lock::RwLock;
-use collab_entity::{CollabType, EncodedCollab};
+use collab_entity::CollabType;
use collab_folder::{Folder, FolderNotify};
use collab_integrate::CollabKVDB;
use flowy_error::{FlowyError, FlowyResult};
use std::sync::{Arc, Weak};
-use tokio::task::spawn_blocking;
-use tracing::{error, event, info, Level};
+use tracing::{event, info, Level};
use uuid::Uuid;
impl FolderManager {
@@ -28,6 +27,7 @@ impl FolderManager {
workspace_id,
initial_data
);
+ let _ = self.folder_ready_notifier.send_replace(false);
if let Some(old_folder) = self.mutex_folder.swap(None) {
let old_folder = old_folder.read().await;
@@ -111,17 +111,11 @@ impl FolderManager {
let folder_state_rx = {
let folder = folder.read().await;
- let folder_state_rx = folder.subscribe_sync_state();
- let index_content_rx = folder.subscribe_index_content();
- self
- .folder_indexer
- .set_index_content_receiver(index_content_rx, *workspace_id)
- .await;
- self.handle_index_folder(*workspace_id, &folder).await;
- folder_state_rx
+ folder.subscribe_sync_state()
};
self.mutex_folder.store(Some(folder.clone()));
+ let _ = self.folder_ready_notifier.send_replace(true);
let weak_mutex_folder = Arc::downgrade(&folder);
subscribe_folder_sync_state_changed(*workspace_id, folder_state_rx, Arc::downgrade(&self.user));
@@ -138,16 +132,6 @@ impl FolderManager {
Arc::downgrade(&self.user),
);
- let weak_folder_indexer = Arc::downgrade(&self.folder_indexer);
- let workspace_id = *workspace_id;
- tokio::spawn(async move {
- if let Some(folder_indexer) = weak_folder_indexer.upgrade() {
- if let Err(err) = folder_indexer.initialize(&workspace_id).await {
- error!("Failed to initialize folder indexer: {:?}", err);
- }
- }
- });
-
Ok(())
}
@@ -176,51 +160,4 @@ impl FolderManager {
.await?;
Ok(folder)
}
-
- async fn handle_index_folder(&self, workspace_id: Uuid, folder: &Folder) {
- let mut index_all = true;
-
- let encoded_collab = self
- .store_preferences
- .get_object::<EncodedCollab>(workspace_id.to_string().as_str());
-
- if let Some(encoded_collab) = encoded_collab {
- if let Ok(changes) = folder.calculate_view_changes(encoded_collab) {
- let folder_indexer = self.folder_indexer.clone();
-
- let views = folder.get_all_views();
- if !changes.is_empty() && !views.is_empty() {
- spawn_blocking(move || {
- // We index the changes
- folder_indexer.index_view_changes(views, changes, workspace_id);
- });
- index_all = false;
- }
- }
- }
-
- if index_all {
- let views = folder.get_all_views();
- let folder_indexer = self.folder_indexer.clone();
- let _ = folder_indexer
- .remove_indices_for_workspace(workspace_id)
- .await;
- // We spawn a blocking task to index all views in the folder
- spawn_blocking(move || {
- folder_indexer.index_all_views(views, workspace_id);
- });
- }
-
- self.save_collab_to_preferences(folder);
- }
-
- fn save_collab_to_preferences(&self, folder: &Folder) {
- if let Some(workspace_id) = folder.get_workspace_id() {
- let encoded_collab = folder.encode_collab();
-
- if let Ok(encoded) = encoded_collab {
- let _ = self.store_preferences.set_object(&workspace_id, &encoded);
- }
- }
- }
}
diff --git a/frontend/rust-lib/flowy-search-pub/src/entities.rs b/frontend/rust-lib/flowy-search-pub/src/entities.rs
index 4cc625af46..606340ccc8 100644
--- a/frontend/rust-lib/flowy-search-pub/src/entities.rs
+++ b/frontend/rust-lib/flowy-search-pub/src/entities.rs
@@ -1,12 +1,11 @@
use std::sync::Arc;
use collab::core::collab::IndexContentReceiver;
-use collab_folder::{folder_diff::FolderViewChange, View, ViewIcon, ViewLayout};
-use flowy_error::FlowyError;
+use collab_folder::{View, ViewIcon, ViewLayout};
use lib_infra::async_trait::async_trait;
use uuid::Uuid;
-pub struct IndexableData {
+pub struct ViewObserveData {
pub id: String,
pub data: String,
pub icon: Option<ViewIcon>,
@@ -14,9 +13,9 @@ pub struct IndexableData {
pub workspace_id: Uuid,
}
-impl IndexableData {
+impl ViewObserveData {
pub fn from_view(view: Arc<View>, workspace_id: Uuid) -> Self {
- IndexableData {
+ ViewObserveData {
id: view.id.clone(),
data: view.name.clone(),
icon: view.icon.clone(),
@@ -27,25 +26,6 @@ impl IndexableData {
}
#[async_trait]
-pub trait IndexManager: Send + Sync {
- async fn set_index_content_receiver(&self, rx: IndexContentReceiver, workspace_id: Uuid);
- async fn add_index(&self, data: IndexableData) -> Result<(), FlowyError>;
- async fn update_index(&self, data: IndexableData) -> Result<(), FlowyError>;
- async fn remove_indices(&self, ids: Vec<String>) -> Result<(), FlowyError>;
- async fn remove_indices_for_workspace(&self, workspace_id: Uuid) -> Result<(), FlowyError>;
- async fn is_indexed(&self) -> bool;
-}
-
-#[async_trait]
-pub trait FolderIndexManager: IndexManager {
- async fn initialize(&self, workspace_id: &Uuid) -> Result<(), FlowyError>;
-
- fn index_all_views(&self, views: Vec<Arc<View>>, workspace_id: Uuid);
-
- fn index_view_changes(
- &self,
- views: Vec<Arc<View>>,
- changes: Vec<FolderViewChange>,
- workspace_id: Uuid,
- );
+pub trait FolderViewObserver: Send + Sync {
+ async fn set_observer_rx(&self, rx: IndexContentReceiver);
}
diff --git a/frontend/rust-lib/flowy-search/Cargo.toml b/frontend/rust-lib/flowy-search/Cargo.toml
index a803ad894f..87dffebca2 100644
--- a/frontend/rust-lib/flowy-search/Cargo.toml
+++ b/frontend/rust-lib/flowy-search/Cargo.toml
@@ -29,7 +29,6 @@ serde_json.workspace = true
tokio = { workspace = true, features = ["full", "rt-multi-thread", "tracing"] }
tracing.workspace = true
derive_builder.workspace = true
-strsim = "0.11.0"
strum_macros = "0.26.1"
tantivy.workspace = true
uuid.workspace = true
@@ -37,6 +36,8 @@ allo-isolate = { version = "^0.1", features = ["catch-unwind"] }
futures.workspace = true
tokio-stream.workspace = true
async-stream = "0.3.6"
+dashmap.workspace = true
+arc-swap.workspace = true
[build-dependencies]
flowy-codegen.workspace = true
diff --git a/frontend/rust-lib/flowy-search/src/document/handler.rs b/frontend/rust-lib/flowy-search/src/document/cloud_search_handler.rs
similarity index 87%
rename from frontend/rust-lib/flowy-search/src/document/handler.rs
rename to frontend/rust-lib/flowy-search/src/document/cloud_search_handler.rs
index 2127ef0d98..ec9a4e1d09 100644
--- a/frontend/rust-lib/flowy-search/src/document/handler.rs
+++ b/frontend/rust-lib/flowy-search/src/document/cloud_search_handler.rs
@@ -3,7 +3,7 @@ use crate::entities::{
SearchResponsePB, SearchSourcePB, SearchSummaryPB,
};
use crate::{
- entities::{ResultIconPB, ResultIconTypePB, SearchFilterPB, SearchResponseItemPB},
+ entities::{ResultIconPB, ResultIconTypePB, SearchResponseItemPB},
services::manager::{SearchHandler, SearchType},
};
use async_stream::stream;
@@ -13,18 +13,17 @@ use flowy_folder::{manager::FolderManager, ViewLayout};
use flowy_search_pub::cloud::{SearchCloudService, SearchResult};
use lib_infra::async_trait::async_trait;
use std::pin::Pin;
-use std::str::FromStr;
use std::sync::Arc;
use tokio_stream::{self, Stream};
use tracing::{trace, warn};
use uuid::Uuid;
-pub struct DocumentSearchHandler {
+pub struct DocumentCloudSearchHandler {
pub cloud_service: Arc<dyn SearchCloudService>,
pub folder_manager: Arc<FolderManager>,
}
-impl DocumentSearchHandler {
+impl DocumentCloudSearchHandler {
pub fn new(
cloud_service: Arc<dyn SearchCloudService>,
folder_manager: Arc<FolderManager>,
@@ -35,38 +34,23 @@ impl DocumentSearchHandler {
}
}
}
+
#[async_trait]
-impl SearchHandler for DocumentSearchHandler {
+impl SearchHandler for DocumentCloudSearchHandler {
fn search_type(&self) -> SearchType {
- SearchType::Document
+ SearchType::DocumentCloud
}
async fn perform_search(
&self,
query: String,
- filter: Option<SearchFilterPB>,
+ workspace_id: &Uuid,
) -> Pin<Box<dyn Stream<Item = FlowyResult<SearchResponsePB>> + Send + 'static>> {
let cloud_service = self.cloud_service.clone();
let folder_manager = self.folder_manager.clone();
+ let workspace_id = *workspace_id;
Box::pin(stream! {
- // Exit early if there is no filter.
- let filter = if let Some(f) = filter {
- f
- } else {
- yield Ok(CreateSearchResultPBArgs::default().build().unwrap());
- return;
- };
-
- // Parse workspace id.
- let workspace_id = match Uuid::from_str(&filter.workspace_id) {
- Ok(id) => id,
- Err(e) => {
- yield Err(e.into());
- return;
- }
- };
-
// Retrieve all available views.
let views = match folder_manager.get_all_views_pb().await {
Ok(views) => views,
@@ -128,7 +112,7 @@ impl SearchHandler for DocumentSearchHandler {
.unwrap(),
);
- if result_items.is_empty() {
+ if summary_input.is_empty() {
return;
}
diff --git a/frontend/rust-lib/flowy-search/src/document/local_search_handler.rs b/frontend/rust-lib/flowy-search/src/document/local_search_handler.rs
new file mode 100644
index 0000000000..c485d5c909
--- /dev/null
+++ b/frontend/rust-lib/flowy-search/src/document/local_search_handler.rs
@@ -0,0 +1,347 @@
+use async_stream::stream;
+use collab_folder::ViewIcon;
+use futures::Stream;
+use std::fs;
+use std::path::PathBuf;
+use std::pin::Pin;
+use std::sync::Weak;
+use tantivy::directory::MmapDirectory;
+use tantivy::schema::Value;
+use tantivy::{doc, Index, IndexReader, IndexWriter, TantivyDocument, Term};
+use tokio::sync::RwLock;
+use tracing::{error, trace, warn};
+use uuid::Uuid;
+
+use crate::entities::{
+ CreateSearchResultPBArgs, LocalSearchResponseItemPB, RepeatedLocalSearchResponseItemPB,
+ ResultIconPB, ResultIconTypePB, SearchResponsePB,
+};
+use crate::schema::LocalSearchTantivySchema;
+use crate::services::manager::{SearchHandler, SearchType};
+use flowy_error::{FlowyError, FlowyResult};
+use lib_infra::async_trait::async_trait;
+
+pub struct DocumentLocalSearchHandler {
+ state: Weak<RwLock<DocumentTantivyState>>,
+}
+
+impl DocumentLocalSearchHandler {
+ pub fn new(state: Weak<RwLock<DocumentTantivyState>>) -> Self {
+ Self { state }
+ }
+}
+
+#[async_trait]
+impl SearchHandler for DocumentLocalSearchHandler {
+ fn search_type(&self) -> SearchType {
+ SearchType::DocumentLocal
+ }
+
+ async fn perform_search(
+ &self,
+ query: String,
+ workspace_id: &Uuid,
+ ) -> Pin<Box<dyn Stream<Item = FlowyResult<SearchResponsePB>> + Send + 'static>> {
+ let workspace_id = *workspace_id;
+ let state = self.state.clone();
+ Box::pin(stream! {
+ match state.upgrade() {
+ None => {
+ yield Ok(
+ CreateSearchResultPBArgs::default()
+ .local_search_result(None)
+ .build()
+ .unwrap(),
+ );
+ },
+ Some(state) => {
+ match state.read().await.search(&workspace_id, &query) {
+ Ok(items) => {
+ trace!("[Tanvity] local document search result: {:?}", items);
+ if items.is_empty() {
+ yield Ok(
+ CreateSearchResultPBArgs::default()
+ .local_search_result(None)
+ .build()
+ .unwrap(),
+ );
+ } else {
+ let search_result = RepeatedLocalSearchResponseItemPB { items };
+ yield Ok(
+ CreateSearchResultPBArgs::default()
+ .local_search_result(Some(search_result))
+ .build()
+ .unwrap(),
+ );
+ }
+ },
+ Err(err) => error!("[Tantivy] Failed to search documents, {:?}", err),
+ }
+ }
+ }
+ })
+ }
+}
+
+/// Holds the Tantivy index state for a workspace's documents.
+pub struct DocumentTantivyState {
+ pub path: PathBuf,
+ pub index: Index,
+ pub schema: LocalSearchTantivySchema,
+ pub writer: IndexWriter,
+ pub reader: IndexReader,
+ pub workspace_id: Uuid,
+ // Cached fields for better performance
+ field_workspace_id: tantivy::schema::Field,
+ field_object_id: tantivy::schema::Field,
+ field_content: tantivy::schema::Field,
+ field_name: tantivy::schema::Field,
+ field_icon: tantivy::schema::Field,
+ field_icon_type: tantivy::schema::Field,
+}
+
+impl DocumentTantivyState {
+ pub fn new(workspace_id: &Uuid, path: PathBuf) -> FlowyResult<Self> {
+ let index_path = path.join(workspace_id.to_string()).join("documents");
+ if !index_path.exists() {
+ fs::create_dir_all(&index_path).map_err(|e| {
+ error!("Failed to create index directory: {:?}", e);
+ FlowyError::internal().with_context("Failed to create folder index")
+ })?;
+ }
+
+ let schema = LocalSearchTantivySchema::new();
+ let dir = MmapDirectory::open(&index_path)?;
+ let index = Index::open_or_create(dir, schema.0.clone())?;
+ let writer = index.writer(15_000_000)?; // 15 MB buffer
+ let reader = index.reader()?;
+
+ // Cache field lookups
+ let field_workspace_id = schema
+ .0
+ .get_field(LocalSearchTantivySchema::WORKSPACE_ID)
+ .map_err(|_| FlowyError::internal().with_context("workspace_id field missing"))?;
+ let field_object_id = schema
+ .0
+ .get_field(LocalSearchTantivySchema::OBJECT_ID)
+ .map_err(|_| FlowyError::internal().with_context("object_id field missing"))?;
+ let field_content = schema
+ .0
+ .get_field(LocalSearchTantivySchema::CONTENT)
+ .map_err(|_| FlowyError::internal().with_context("content field missing"))?;
+ let field_name = schema
+ .0
+ .get_field(LocalSearchTantivySchema::NAME)
+ .map_err(|_| FlowyError::internal().with_context("name field missing"))?;
+ let field_icon = schema
+ .0
+ .get_field(LocalSearchTantivySchema::ICON)
+ .map_err(|_| FlowyError::internal().with_context("icon field missing"))?;
+ let field_icon_type = schema
+ .0
+ .get_field(LocalSearchTantivySchema::ICON_TYPE)
+ .map_err(|_| FlowyError::internal().with_context("icon_type field missing"))?;
+
+ Ok(Self {
+ path,
+ index,
+ schema,
+ writer,
+ reader,
+ workspace_id: *workspace_id,
+ field_workspace_id,
+ field_object_id,
+ field_content,
+ field_name,
+ field_icon,
+ field_icon_type,
+ })
+ }
+
+ pub fn add_document(
+ &mut self,
+ id: &str,
+ content: String,
+ name: String,
+ icon: Option<ViewIcon>,
+ ) -> FlowyResult<()> {
+ trace!("[Tantivy] Adding document with id:{}, name:{}", id, name);
+ let term = Term::from_field_text(self.field_object_id, id);
+ // Delete existing document with same ID
+ self.writer.delete_term(term);
+
+ let (icon, icon_type) = icon.map(|v| (v.value, v.ty as u8)).unwrap_or_default();
+
+ // Create document with cached fields
+ let tantivy_doc = doc!(
+ self.field_workspace_id => self.workspace_id.to_string(),
+ self.field_object_id => id,
+ self.field_content => content,
+ self.field_name => name,
+ self.field_icon => icon,
+ self.field_icon_type => icon_type.to_string()
+ );
+
+ self.writer.add_document(tantivy_doc)?;
+ self.writer.commit()?;
+
+ Ok(())
+ }
+
+ pub fn add_document_metadata(
+ &mut self,
+ id: &str,
+ name: String,
+ icon: Option<ViewIcon>,
+ ) -> FlowyResult<()> {
+ let term = Term::from_field_text(self.field_object_id, id);
+ let searcher = self.reader.searcher();
+ let query =
+ tantivy::query::TermQuery::new(term.clone(), tantivy::schema::IndexRecordOption::Basic);
+
+ // Search for the document
+ let top_docs = searcher.search(&query, &tantivy::collector::TopDocs::with_limit(1))?;
+ let content = if let Some((_score, doc_address)) = top_docs.first() {
+ let retrieved: TantivyDocument = searcher.doc(*doc_address)?;
+ retrieved
+ .get_first(self.field_content)
+ .and_then(|v| v.as_str())
+ .unwrap_or_default()
+ .to_string()
+ } else {
+ String::new()
+ };
+ self.add_document(id, content, name, icon)?;
+ Ok(())
+ }
+
+ pub fn delete_workspace(&mut self, workspace_id: &Uuid) -> FlowyResult<()> {
+ let term = Term::from_field_text(self.field_workspace_id, &workspace_id.to_string());
+ self.writer.delete_term(term);
+ self.writer.commit()?;
+
+ Ok(())
+ }
+
+ /// Delete a document (all fields) matching this `object_id`
+ pub fn delete_document(&mut self, id: &str) -> FlowyResult<()> {
+ trace!("[Tantivy] delete document with id: {}", id);
+ let term = Term::from_field_text(self.field_object_id, id);
+ self.writer.delete_term(term);
+ self.writer.commit()?;
+
+ Ok(())
+ }
+
+ pub fn delete_documents(&mut self, ids: &[String]) -> FlowyResult<()> {
+ trace!("[Tantivy] delete documents with ids: {:?}", ids);
+ for id in ids {
+ let term = Term::from_field_text(self.field_object_id, id);
+ self.writer.delete_term(term);
+ }
+ self.writer.commit()?;
+
+ Ok(())
+ }
+
+ pub fn search(
+ &self,
+ workspace_id: &Uuid,
+ query: &str,
+ ) -> FlowyResult<Vec<LocalSearchResponseItemPB>> {
+ let workspace_id = workspace_id.to_string();
+ let reader = self.reader.clone();
+ let searcher = reader.searcher();
+
+ // Use cached fields for query parser
+ let mut qp = tantivy::query::QueryParser::for_index(
+ &self.index,
+ vec![self.field_content, self.field_name],
+ );
+ // Enable fuzzy matching for name field (better user experience for typos)
+ qp.set_field_fuzzy(self.field_name, true, 2, true);
+
+ let query = qp.parse_query(query)?;
+ let top_docs = searcher.search(&query, &tantivy::collector::TopDocs::with_limit(10))?;
+
+ let mut results = Vec::with_capacity(top_docs.len());
+ let mut seen_ids = std::collections::HashSet::new();
+
+ for (_score, doc_address) in top_docs {
+ let retrieved: TantivyDocument = searcher.doc(doc_address)?;
+ // Pull out each stored field using cached field references
+ let workspace_id_str = retrieved
+ .get_first(self.field_workspace_id)
+ .and_then(|v| v.as_str())
+ .unwrap_or_default()
+ .to_string();
+
+ if workspace_id != workspace_id_str {
+ warn!(
+ "[Tantivy] Document workspace_id mismatch: {} != {}",
+ workspace_id, workspace_id_str
+ );
+ continue;
+ }
+
+ let object_id = retrieved
+ .get_first(self.field_object_id)
+ .and_then(|v| v.as_str())
+ .unwrap_or_default()
+ .to_string();
+
+ // Skip records with empty object_id and workspace_id
+ if object_id.is_empty() && workspace_id_str.is_empty() {
+ continue;
+ }
+
+ // Skip duplicate records based on object_id
+ if !seen_ids.insert(object_id.clone()) {
+ continue;
+ }
+
+ let name = retrieved
+ .get_first(self.field_name)
+ .and_then(|v| v.as_str())
+ .unwrap_or_default()
+ .to_string();
+
+ // Get icon value and type
+ let icon = {
+ let icon_value = retrieved
+ .get_first(self.field_icon)
+ .and_then(|v| v.as_str())
+ .unwrap_or_default()
+ .to_string();
+
+ let icon_type_str = retrieved
+ .get_first(self.field_icon_type)
+ .and_then(|v| v.as_str())
+ .unwrap_or_default();
+
+ let icon_type: ResultIconTypePB = match icon_type_str.parse::<i64>() {
+ Ok(val) => val.into(),
+ Err(_) => ResultIconTypePB::default(),
+ };
+
+ if icon_value.is_empty() {
+ None
+ } else {
+ Some(ResultIconPB {
+ ty: icon_type,
+ value: icon_value,
+ })
+ }
+ };
+
+ results.push(LocalSearchResponseItemPB {
+ id: object_id,
+ display_name: name,
+ icon,
+ workspace_id: workspace_id_str,
+ });
+ }
+
+ Ok(results)
+ }
+}
diff --git a/frontend/rust-lib/flowy-search/src/document/mod.rs b/frontend/rust-lib/flowy-search/src/document/mod.rs
index 062ae9d9be..c083b36aea 100644
--- a/frontend/rust-lib/flowy-search/src/document/mod.rs
+++ b/frontend/rust-lib/flowy-search/src/document/mod.rs
@@ -1 +1,2 @@
-pub mod handler;
+pub mod cloud_search_handler;
+pub mod local_search_handler;
diff --git a/frontend/rust-lib/flowy-search/src/entities/query.rs b/frontend/rust-lib/flowy-search/src/entities/query.rs
index 65c92ebed0..97342eef66 100644
--- a/frontend/rust-lib/flowy-search/src/entities/query.rs
+++ b/frontend/rust-lib/flowy-search/src/entities/query.rs
@@ -1,7 +1,5 @@
use flowy_derive::ProtoBuf;
-use super::SearchFilterPB;
-
#[derive(Eq, PartialEq, ProtoBuf, Default, Debug, Clone)]
pub struct SearchQueryPB {
#[pb(index = 1)]
@@ -10,12 +8,9 @@ pub struct SearchQueryPB {
#[pb(index = 2, one_of)]
pub limit: Option<i64>,
- #[pb(index = 3, one_of)]
- pub filter: Option<SearchFilterPB>,
-
- #[pb(index = 4)]
+ #[pb(index = 3)]
pub search_id: String,
- #[pb(index = 5)]
+ #[pb(index = 4)]
pub stream_port: i64,
}
diff --git a/frontend/rust-lib/flowy-search/src/event_handler.rs b/frontend/rust-lib/flowy-search/src/event_handler.rs
index d79a719f6f..750fc530b5 100644
--- a/frontend/rust-lib/flowy-search/src/event_handler.rs
+++ b/frontend/rust-lib/flowy-search/src/event_handler.rs
@@ -1,9 +1,9 @@
use std::sync::{Arc, Weak};
+use crate::{entities::SearchQueryPB, services::manager::SearchManager};
use flowy_error::{FlowyError, FlowyResult};
use lib_dispatch::prelude::{AFPluginData, AFPluginState};
-
-use crate::{entities::SearchQueryPB, services::manager::SearchManager};
+use lib_infra::util::timestamp;
fn upgrade_manager(
search_manager: AFPluginState<Weak<SearchManager>>,
@@ -15,19 +15,15 @@ fn upgrade_manager(
}
#[tracing::instrument(level = "debug", skip(manager), err)]
-pub(crate) async fn search_handler(
+pub(crate) async fn stream_search_handler(
data: AFPluginData<SearchQueryPB>,
manager: AFPluginState<Weak<SearchManager>>,
) -> Result<(), FlowyError> {
let query = data.into_inner();
let manager = upgrade_manager(manager)?;
+ let search_id = query.search_id.parse::<i64>().unwrap_or(timestamp());
manager
- .perform_search(
- query.search,
- query.stream_port,
- query.filter,
- query.search_id,
- )
+ .perform_search(query.search, query.stream_port, search_id)
.await;
Ok(())
diff --git a/frontend/rust-lib/flowy-search/src/event_map.rs b/frontend/rust-lib/flowy-search/src/event_map.rs
index 7ab8838633..a0bb009ad4 100644
--- a/frontend/rust-lib/flowy-search/src/event_map.rs
+++ b/frontend/rust-lib/flowy-search/src/event_map.rs
@@ -4,18 +4,18 @@ use strum_macros::Display;
use flowy_derive::{Flowy_Event, ProtoBuf_Enum};
use lib_dispatch::prelude::*;
-use crate::{event_handler::search_handler, services::manager::SearchManager};
+use crate::{event_handler::stream_search_handler, services::manager::SearchManager};
pub fn init(search_manager: Weak<SearchManager>) -> AFPlugin {
AFPlugin::new()
.state(search_manager)
.name(env!("CARGO_PKG_NAME"))
- .event(SearchEvent::Search, search_handler)
+ .event(SearchEvent::StreamSearch, stream_search_handler)
}
#[derive(Clone, Copy, PartialEq, Eq, Debug, Display, Hash, ProtoBuf_Enum, Flowy_Event)]
#[event_err = "FlowyError"]
pub enum SearchEvent {
#[event(input = "SearchQueryPB")]
- Search = 0,
+ StreamSearch = 0,
}
diff --git a/frontend/rust-lib/flowy-search/src/folder/entities.rs b/frontend/rust-lib/flowy-search/src/folder/entities.rs
deleted file mode 100644
index 1bb763b4a6..0000000000
--- a/frontend/rust-lib/flowy-search/src/folder/entities.rs
+++ /dev/null
@@ -1,32 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-use crate::entities::{LocalSearchResponseItemPB, ResultIconPB};
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct FolderIndexData {
- pub id: String,
- pub title: String,
- pub icon: String,
- pub icon_ty: i64,
- pub workspace_id: String,
-}
-
-impl From<FolderIndexData> for LocalSearchResponseItemPB {
- fn from(data: FolderIndexData) -> Self {
- let icon = if data.icon.is_empty() {
- None
- } else {
- Some(ResultIconPB {
- ty: data.icon_ty.into(),
- value: data.icon,
- })
- };
-
- Self {
- id: data.id,
- display_name: data.title,
- icon,
- workspace_id: data.workspace_id,
- }
- }
-}
diff --git a/frontend/rust-lib/flowy-search/src/folder/handler.rs b/frontend/rust-lib/flowy-search/src/folder/handler.rs
deleted file mode 100644
index e21ce1c98c..0000000000
--- a/frontend/rust-lib/flowy-search/src/folder/handler.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-use super::indexer::FolderIndexManagerImpl;
-use crate::entities::{
- CreateSearchResultPBArgs, RepeatedLocalSearchResponseItemPB, SearchFilterPB, SearchResponsePB,
-};
-use crate::services::manager::{SearchHandler, SearchType};
-use async_stream::stream;
-use flowy_error::FlowyResult;
-use lib_infra::async_trait::async_trait;
-use std::pin::Pin;
-use std::sync::Arc;
-use tokio_stream::{self, Stream};
-
-pub struct FolderSearchHandler {
- pub index_manager: Arc<FolderIndexManagerImpl>,
-}
-
-impl FolderSearchHandler {
- pub fn new(index_manager: Arc<FolderIndexManagerImpl>) -> Self {
- Self { index_manager }
- }
-}
-
-#[async_trait]
-impl SearchHandler for FolderSearchHandler {
- fn search_type(&self) -> SearchType {
- SearchType::Folder
- }
-
- async fn perform_search(
- &self,
- query: String,
- filter: Option<SearchFilterPB>,
- ) -> Pin<Box<dyn Stream<Item = FlowyResult<SearchResponsePB>> + Send + 'static>> {
- let index_manager = self.index_manager.clone();
-
- Box::pin(stream! {
- // Perform search (if search() returns a Result)
- let mut items = match index_manager.search(query).await {
- Ok(items) => items,
- Err(err) => {
- yield Err(err);
- return;
- }
- };
-
- if let Some(filter) = filter {
- items.retain(|result| result.workspace_id == filter.workspace_id);
- }
-
- // Build the search result.
- let search_result = RepeatedLocalSearchResponseItemPB {items};
- yield Ok(CreateSearchResultPBArgs::default().local_search_result(Some(search_result)).build().unwrap())
- })
- }
-}
diff --git a/frontend/rust-lib/flowy-search/src/folder/indexer.rs b/frontend/rust-lib/flowy-search/src/folder/indexer.rs
deleted file mode 100644
index 59622852d5..0000000000
--- a/frontend/rust-lib/flowy-search/src/folder/indexer.rs
+++ /dev/null
@@ -1,397 +0,0 @@
-use super::entities::FolderIndexData;
-use crate::entities::{LocalSearchResponseItemPB, ResultIconTypePB};
-use crate::folder::schema::{
- FolderSchema, FOLDER_ICON_FIELD_NAME, FOLDER_ICON_TY_FIELD_NAME, FOLDER_ID_FIELD_NAME,
- FOLDER_TITLE_FIELD_NAME, FOLDER_WORKSPACE_ID_FIELD_NAME,
-};
-use collab::core::collab::{IndexContent, IndexContentReceiver};
-use collab_folder::{folder_diff::FolderViewChange, View, ViewIcon, ViewIndexContent, ViewLayout};
-use flowy_error::{FlowyError, FlowyResult};
-use flowy_search_pub::entities::{FolderIndexManager, IndexManager, IndexableData};
-use flowy_user::services::authenticate_user::AuthenticateUser;
-use lib_infra::async_trait::async_trait;
-use std::path::PathBuf;
-use std::sync::{Arc, Weak};
-use std::{collections::HashMap, fs};
-use tantivy::{
- collector::TopDocs, directory::MmapDirectory, doc, query::QueryParser, schema::Field, Document,
- Index, IndexReader, IndexWriter, TantivyDocument, TantivyError, Term,
-};
-use tokio::sync::RwLock;
-use tracing::{error, info};
-use uuid::Uuid;
-
-pub struct TantivyState {
- pub path: PathBuf,
- pub index: Index,
- pub folder_schema: FolderSchema,
- pub index_reader: IndexReader,
- pub index_writer: IndexWriter,
-}
-
-impl Drop for TantivyState {
- fn drop(&mut self) {
- tracing::trace!("Dropping TantivyState at {:?}", self.path);
- }
-}
-
-#[derive(Clone)]
-pub struct FolderIndexManagerImpl {
- auth_user: Weak<AuthenticateUser>,
- state: Arc<RwLock<Option<TantivyState>>>,
-}
-
-impl FolderIndexManagerImpl {
- pub fn new(auth_user: Weak<AuthenticateUser>) -> Self {
- Self {
- auth_user,
- state: Arc::new(RwLock::new(None)),
- }
- }
-
- async fn with_writer<F, R>(&self, f: F) -> FlowyResult<R>
- where
- F: FnOnce(&mut IndexWriter, &FolderSchema) -> FlowyResult<R>,
- {
- let mut lock = self.state.write().await;
- if let Some(ref mut state) = *lock {
- f(&mut state.index_writer, &state.folder_schema)
- } else {
- Err(FlowyError::internal().with_context("Index not initialized. Call initialize first"))
- }
- }
-
- /// Initializes the state using the workspace directory.
- async fn initialize(&self, workspace_id: &Uuid) -> FlowyResult<()> {
- if let Some(state) = self.state.write().await.take() {
- info!("Re-initializing folder indexer");
- drop(state);
- }
-
- // Since the directory lock may not be immediately released,
- // a workaround is implemented by waiting for 3 seconds before proceeding further. This delay helps
- // to avoid errors related to trying to open an index directory while an IndexWriter is still active.
- //
- // Also, we don't need to initialize the indexer immediately.
- tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
-
- let auth_user = self
- .auth_user
- .upgrade()
- .ok_or_else(|| FlowyError::internal().with_context("AuthenticateUser is not available"))?;
-
- let index_path = auth_user.get_index_path()?.join(workspace_id.to_string());
- if !index_path.exists() {
- fs::create_dir_all(&index_path).map_err(|e| {
- error!("Failed to create folder index directory: {:?}", e);
- FlowyError::internal().with_context("Failed to create folder index")
- })?;
- }
-
- info!("Folder indexer initialized at: {:?}", index_path);
- let folder_schema = FolderSchema::new();
- let dir = MmapDirectory::open(index_path.clone())?;
- let index = Index::open_or_create(dir, folder_schema.schema.clone())?;
- let index_reader = index.reader()?;
-
- let index_writer = match index.writer::<_>(50_000_000) {
- Ok(index_writer) => index_writer,
- Err(err) => {
- if let TantivyError::LockFailure(_, _) = err {
- error!(
- "Failed to acquire lock for index writer: {:?}, retry later",
- err
- );
- tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
- }
- index.writer::<_>(50_000_000)?
- },
- };
-
- *self.state.write().await = Some(TantivyState {
- path: index_path,
- index,
- folder_schema,
- index_reader,
- index_writer,
- });
-
- Ok(())
- }
-
- fn extract_icon(
- &self,
- view_icon: Option<ViewIcon>,
- view_layout: ViewLayout,
- ) -> (Option<String>, i64) {
- let icon_ty: i64;
- let icon: Option<String>;
-
- if view_icon.clone().is_some_and(|v| !v.value.is_empty()) {
- let view_icon = view_icon.unwrap();
- let result_icon_ty: ResultIconTypePB = view_icon.ty.into();
- icon_ty = result_icon_ty.into();
- icon = Some(view_icon.value);
- } else {
- icon_ty = ResultIconTypePB::Icon.into();
- let layout_ty = view_layout as i64;
- icon = Some(layout_ty.to_string());
- }
- (icon, icon_ty)
- }
-
- /// Simple implementation to index all given data by spawning async tasks.
- fn index_all(&self, data_vec: Vec<IndexableData>) -> Result<(), FlowyError> {
- for data in data_vec {
- let indexer = self.clone();
- tokio::spawn(async move {
- let _ = indexer.add_index(data).await;
- });
- }
- Ok(())
- }
-
- /// Searches the index using the given query string.
- pub async fn search(&self, query: String) -> Result<Vec<LocalSearchResponseItemPB>, FlowyError> {
- let lock = self.state.read().await;
- let state = lock
- .as_ref()
- .ok_or_else(FlowyError::folder_index_manager_unavailable)?;
- let schema = &state.folder_schema;
- let index = &state.index;
- let reader = &state.index_reader;
-
- let title_field = schema.schema.get_field(FOLDER_TITLE_FIELD_NAME)?;
- let mut parser = QueryParser::for_index(index, vec![title_field]);
- parser.set_field_fuzzy(title_field, true, 2, true);
-
- let built_query = parser.parse_query(&query)?;
- let searcher = reader.searcher();
- let top_docs = searcher.search(&built_query, &TopDocs::with_limit(10))?;
-
- let mut results = Vec::new();
- for (_score, doc_address) in top_docs {
- let doc: TantivyDocument = searcher.doc(doc_address)?;
- let named_doc = doc.to_named_doc(&schema.schema);
- let mut content = HashMap::new();
- for (k, v) in named_doc.0 {
- content.insert(k, v[0].clone());
- }
- if !content.is_empty() {
- let s = serde_json::to_string(&content)?;
- let result: LocalSearchResponseItemPB = serde_json::from_str::<FolderIndexData>(&s)?.into();
- results.push(result);
- }
- }
-
- Ok(results)
- }
-}
-
-#[async_trait]
-impl IndexManager for FolderIndexManagerImpl {
- async fn set_index_content_receiver(&self, mut rx: IndexContentReceiver, workspace_id: Uuid) {
- let indexer = self.clone();
- let wid = workspace_id;
- tokio::spawn(async move {
- while let Ok(msg) = rx.recv().await {
- match msg {
- IndexContent::Create(value) => match serde_json::from_value::<ViewIndexContent>(value) {
- Ok(view) => {
- let _ = indexer
- .add_index(IndexableData {
- id: view.id,
- data: view.name,
- icon: view.icon,
- layout: view.layout,
- workspace_id: wid,
- })
- .await;
- },
- Err(err) => tracing::error!("FolderIndexManager error deserialize (create): {:?}", err),
- },
- IndexContent::Update(value) => match serde_json::from_value::<ViewIndexContent>(value) {
- Ok(view) => {
- let _ = indexer
- .update_index(IndexableData {
- id: view.id,
- data: view.name,
- icon: view.icon,
- layout: view.layout,
- workspace_id: wid,
- })
- .await;
- },
- Err(err) => error!("FolderIndexManager error deserialize (update): {:?}", err),
- },
- IndexContent::Delete(ids) => {
- if let Err(e) = indexer.remove_indices(ids).await {
- error!("FolderIndexManager error (delete): {:?}", e);
- }
- },
- }
- }
- });
- }
-
- async fn add_index(&self, data: IndexableData) -> Result<(), FlowyError> {
- let (icon, icon_ty) = self.extract_icon(data.icon, data.layout);
- self
- .with_writer(|index_writer, folder_schema| {
- let (id_field, title_field, icon_field, icon_ty_field, workspace_id_field) =
- get_schema_fields(folder_schema)?;
- let _ = index_writer.add_document(doc![
- id_field => data.id,
- title_field => data.data,
- icon_field => icon.unwrap_or_default(),
- icon_ty_field => icon_ty,
- workspace_id_field => data.workspace_id.to_string(),
- ]);
- index_writer.commit()?;
- Ok(())
- })
- .await?;
-
- Ok(())
- }
-
- async fn update_index(&self, data: IndexableData) -> Result<(), FlowyError> {
- self
- .with_writer(|index_writer, folder_schema| {
- let (id_field, title_field, icon_field, icon_ty_field, workspace_id_field) =
- get_schema_fields(folder_schema)?;
- let delete_term = Term::from_field_text(id_field, &data.id);
- index_writer.delete_term(delete_term);
-
- let (icon, icon_ty) = self.extract_icon(data.icon, data.layout);
- let _ = index_writer.add_document(doc![
- id_field => data.id,
- title_field => data.data,
- icon_field => icon.unwrap_or_default(),
- icon_ty_field => icon_ty,
- workspace_id_field => data.workspace_id.to_string(),
- ]);
-
- index_writer.commit()?;
- Ok(())
- })
- .await?;
-
- Ok(())
- }
-
- async fn remove_indices(&self, ids: Vec<String>) -> Result<(), FlowyError> {
- self
- .with_writer(|index_writer, folder_schema| {
- let id_field = folder_schema.schema.get_field(FOLDER_ID_FIELD_NAME)?;
- for id in ids {
- let delete_term = Term::from_field_text(id_field, &id);
- index_writer.delete_term(delete_term);
- }
-
- index_writer.commit()?;
- Ok(())
- })
- .await?;
-
- Ok(())
- }
-
- async fn remove_indices_for_workspace(&self, workspace_id: Uuid) -> Result<(), FlowyError> {
- self
- .with_writer(|index_writer, folder_schema| {
- let id_field = folder_schema
- .schema
- .get_field(FOLDER_WORKSPACE_ID_FIELD_NAME)?;
-
- let delete_term = Term::from_field_text(id_field, &workspace_id.to_string());
- index_writer.delete_term(delete_term);
- index_writer.commit()?;
- Ok(())
- })
- .await?;
- Ok(())
- }
-
- async fn is_indexed(&self) -> bool {
- let lock = self.state.read().await;
- if let Some(ref state) = *lock {
- state.index_reader.searcher().num_docs() > 0
- } else {
- false
- }
- }
-}
-
-#[async_trait]
-impl FolderIndexManager for FolderIndexManagerImpl {
- async fn initialize(&self, workspace_id: &Uuid) -> Result<(), FlowyError> {
- self.initialize(workspace_id).await?;
- Ok(())
- }
-
- fn index_all_views(&self, views: Vec<Arc<View>>, workspace_id: Uuid) {
- let indexable_data = views
- .into_iter()
- .map(|view| IndexableData::from_view(view, workspace_id))
- .collect();
- let _ = self.index_all(indexable_data);
- }
-
- fn index_view_changes(
- &self,
- views: Vec<Arc<View>>,
- changes: Vec<FolderViewChange>,
- workspace_id: Uuid,
- ) {
- let mut views_iter = views.into_iter();
- for change in changes {
- match change {
- FolderViewChange::Inserted { view_id } => {
- if let Some(view) = views_iter.find(|view| view.id == view_id) {
- let indexable_data = IndexableData::from_view(view, workspace_id);
- let f = self.clone();
- tokio::spawn(async move {
- let _ = f.add_index(indexable_data).await;
- });
- }
- },
- FolderViewChange::Updated { view_id } => {
- if let Some(view) = views_iter.find(|view| view.id == view_id) {
- let indexable_data = IndexableData::from_view(view, workspace_id);
- let f = self.clone();
- tokio::spawn(async move {
- let _ = f.update_index(indexable_data).await;
- });
- }
- },
- FolderViewChange::Deleted { view_ids } => {
- let f = self.clone();
- tokio::spawn(async move {
- let _ = f.remove_indices(view_ids).await;
- });
- },
- }
- }
- }
-}
-
-fn get_schema_fields(
- folder_schema: &FolderSchema,
-) -> Result<(Field, Field, Field, Field, Field), FlowyError> {
- let id_field = folder_schema.schema.get_field(FOLDER_ID_FIELD_NAME)?;
- let title_field = folder_schema.schema.get_field(FOLDER_TITLE_FIELD_NAME)?;
- let icon_field = folder_schema.schema.get_field(FOLDER_ICON_FIELD_NAME)?;
- let icon_ty_field = folder_schema.schema.get_field(FOLDER_ICON_TY_FIELD_NAME)?;
- let workspace_id_field = folder_schema
- .schema
- .get_field(FOLDER_WORKSPACE_ID_FIELD_NAME)?;
-
- Ok((
- id_field,
- title_field,
- icon_field,
- icon_ty_field,
- workspace_id_field,
- ))
-}
diff --git a/frontend/rust-lib/flowy-search/src/folder/mod.rs b/frontend/rust-lib/flowy-search/src/folder/mod.rs
deleted file mode 100644
index 26d1058ef0..0000000000
--- a/frontend/rust-lib/flowy-search/src/folder/mod.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-pub mod entities;
-pub mod handler;
-pub mod indexer;
-pub mod schema;
diff --git a/frontend/rust-lib/flowy-search/src/folder/schema.rs b/frontend/rust-lib/flowy-search/src/folder/schema.rs
deleted file mode 100644
index cb79782fe5..0000000000
--- a/frontend/rust-lib/flowy-search/src/folder/schema.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-use tantivy::schema::Schema;
-
-pub const FOLDER_ID_FIELD_NAME: &str = "id";
-pub const FOLDER_TITLE_FIELD_NAME: &str = "title";
-pub const FOLDER_ICON_FIELD_NAME: &str = "icon";
-pub const FOLDER_ICON_TY_FIELD_NAME: &str = "icon_ty";
-pub const FOLDER_WORKSPACE_ID_FIELD_NAME: &str = "workspace_id";
-
-#[derive(Clone)]
-pub struct FolderSchema {
- pub schema: Schema,
-}
-
-/// Do not change the schema after the index has been created.
-/// Changing field_options or fields, will result in the schema being different
-/// from previously created index, causing tantivy to panic and search to stop functioning.
-///
-/// If you need to change the schema, create a migration that removes the old index,
-/// and creates a new one with the new schema.
-///
-impl FolderSchema {
- pub fn new() -> Self {
- let mut schema_builder = Schema::builder();
- schema_builder.add_text_field(
- FOLDER_ID_FIELD_NAME,
- tantivy::schema::STRING | tantivy::schema::STORED,
- );
- schema_builder.add_text_field(
- FOLDER_TITLE_FIELD_NAME,
- tantivy::schema::TEXT | tantivy::schema::STORED,
- );
- schema_builder.add_text_field(
- FOLDER_ICON_FIELD_NAME,
- tantivy::schema::TEXT | tantivy::schema::STORED,
- );
- schema_builder.add_i64_field(FOLDER_ICON_TY_FIELD_NAME, tantivy::schema::STORED);
- schema_builder.add_text_field(
- FOLDER_WORKSPACE_ID_FIELD_NAME,
- tantivy::schema::TEXT | tantivy::schema::STORED,
- );
-
- let schema = schema_builder.build();
-
- Self { schema }
- }
-}
-
-impl Default for FolderSchema {
- fn default() -> Self {
- Self::new()
- }
-}
diff --git a/frontend/rust-lib/flowy-search/src/lib.rs b/frontend/rust-lib/flowy-search/src/lib.rs
index 820a6d9cb3..fe2fc0f876 100644
--- a/frontend/rust-lib/flowy-search/src/lib.rs
+++ b/frontend/rust-lib/flowy-search/src/lib.rs
@@ -2,6 +2,6 @@ pub mod document;
pub mod entities;
pub mod event_handler;
pub mod event_map;
-pub mod folder;
pub mod protobuf;
+mod schema;
pub mod services;
diff --git a/frontend/rust-lib/flowy-search/src/schema.rs b/frontend/rust-lib/flowy-search/src/schema.rs
new file mode 100644
index 0000000000..36404781d4
--- /dev/null
+++ b/frontend/rust-lib/flowy-search/src/schema.rs
@@ -0,0 +1,30 @@
+use tantivy::schema::{Schema, STORED, STRING, TEXT};
+
+/// Wraps the schema and exposes field‐name constants
+pub struct LocalSearchTantivySchema(pub Schema);
+
+impl Default for LocalSearchTantivySchema {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl LocalSearchTantivySchema {
+ pub const WORKSPACE_ID: &'static str = "workspace_id";
+ pub const OBJECT_ID: &'static str = "object_id";
+ pub const CONTENT: &'static str = "content";
+ pub const NAME: &'static str = "name";
+ pub const ICON: &'static str = "icon";
+ pub const ICON_TYPE: &'static str = "icon_ty";
+
+ pub fn new() -> Self {
+ let mut builder = Schema::builder();
+ builder.add_text_field(Self::WORKSPACE_ID, STRING | STORED);
+ builder.add_text_field(Self::OBJECT_ID, STRING | STORED);
+ builder.add_text_field(Self::CONTENT, TEXT | STORED);
+ builder.add_text_field(Self::NAME, TEXT | STORED);
+ builder.add_text_field(Self::ICON, TEXT | STORED);
+ builder.add_text_field(Self::ICON_TYPE, STRING | STORED);
+ LocalSearchTantivySchema(builder.build())
+ }
+}
diff --git a/frontend/rust-lib/flowy-search/src/services/manager.rs b/frontend/rust-lib/flowy-search/src/services/manager.rs
index f39c73dc4c..b054448fb3 100644
--- a/frontend/rust-lib/flowy-search/src/services/manager.rs
+++ b/frontend/rust-lib/flowy-search/src/services/manager.rs
@@ -1,18 +1,23 @@
-use crate::entities::{SearchFilterPB, SearchResponsePB, SearchStatePB};
+use crate::document::local_search_handler::{DocumentLocalSearchHandler, DocumentTantivyState};
+use crate::entities::{SearchResponsePB, SearchStatePB};
use allo_isolate::Isolate;
+use arc_swap::ArcSwapOption;
+use dashmap::DashMap;
use flowy_error::FlowyResult;
use lib_infra::async_trait::async_trait;
use lib_infra::isolate_stream::{IsolateSink, SinkExt};
-use std::collections::HashMap;
use std::pin::Pin;
-use std::sync::Arc;
+use std::sync::{Arc, Weak};
+use tokio::sync::RwLock;
use tokio_stream::{self, Stream, StreamExt};
-use tracing::{error, trace};
+use tracing::{error, info, trace};
+use uuid::Uuid;
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub enum SearchType {
Folder,
- Document,
+ DocumentCloud,
+ DocumentLocal,
}
#[async_trait]
@@ -24,7 +29,7 @@ pub trait SearchHandler: Send + Sync + 'static {
async fn perform_search(
&self,
query: String,
- filter: Option<SearchFilterPB>,
+ workspace_id: &Uuid,
) -> Pin<Box<dyn Stream<Item = FlowyResult<SearchResponsePB>> + Send + 'static>>;
}
@@ -33,90 +38,153 @@ pub trait SearchHandler: Send + Sync + 'static {
/// to the client until the query has been fully completed.
///
pub struct SearchManager {
- pub handlers: HashMap<SearchType, Arc<dyn SearchHandler>>,
- current_search: Arc<tokio::sync::Mutex<Option<String>>>,
+ handlers: Arc<DashMap<SearchType, Arc<dyn SearchHandler>>>,
+ current_search: Arc<tokio::sync::Mutex<Option<i64>>>,
+ workspace_id: ArcSwapOption<Uuid>,
}
impl SearchManager {
pub fn new(handlers: Vec<Arc<dyn SearchHandler>>) -> Self {
- let handlers: HashMap<SearchType, Arc<dyn SearchHandler>> = handlers
+ let handlers: DashMap<SearchType, Arc<dyn SearchHandler>> = handlers
.into_iter()
.map(|handler| (handler.search_type(), handler))
.collect();
Self {
- handlers,
+ handlers: Arc::new(handlers),
current_search: Arc::new(tokio::sync::Mutex::new(None)),
+ workspace_id: Default::default(),
}
}
- pub fn get_handler(&self, search_type: SearchType) -> Option<&Arc<dyn SearchHandler>> {
- self.handlers.get(&search_type)
+ pub fn get_handler(&self, search_type: SearchType) -> Option<Arc<dyn SearchHandler>> {
+ self.handlers.get(&search_type).map(|h| h.value().clone())
}
- pub async fn perform_search(
+ fn create_local_document_search(&self, state: Option<Weak<RwLock<DocumentTantivyState>>>) {
+ if let Some(state) = state {
+ let handler = DocumentLocalSearchHandler::new(state);
+ info!("[Tanvity] create local document search handler");
+ self
+ .handlers
+ .insert(SearchType::DocumentLocal, Arc::new(handler));
+ } else {
+ error!("[Tanvity] Failed to create local document search handler");
+ }
+ }
+
+ pub async fn on_launch_if_authenticated(
&self,
- query: String,
- stream_port: i64,
- filter: Option<SearchFilterPB>,
- search_id: String,
+ workspace_id: &Uuid,
+ state: Option<Weak<RwLock<DocumentTantivyState>>>,
) {
- // Cancel previous search by updating current_search
- *self.current_search.lock().await = Some(search_id.clone());
+ self.workspace_id.store(Some(Arc::new(*workspace_id)));
+ self.create_local_document_search(state);
+ }
+
+ pub async fn initialize_after_sign_in(
+ &self,
+ workspace_id: &Uuid,
+ state: Option<Weak<RwLock<DocumentTantivyState>>>,
+ ) {
+ self.workspace_id.store(Some(Arc::new(*workspace_id)));
+ self.create_local_document_search(state);
+ }
+
+ pub async fn initialize_after_sign_up(
+ &self,
+ workspace_id: &Uuid,
+ state: Option<Weak<RwLock<DocumentTantivyState>>>,
+ ) {
+ self.workspace_id.store(Some(Arc::new(*workspace_id)));
+ self.create_local_document_search(state);
+ }
+
+ pub async fn initialize_after_open_workspace(
+ &self,
+ workspace_id: &Uuid,
+ state: Option<Weak<RwLock<DocumentTantivyState>>>,
+ ) {
+ self.workspace_id.store(Some(Arc::new(*workspace_id)));
+ self.create_local_document_search(state);
+ }
+ pub async fn perform_search(&self, query: String, stream_port: i64, search_id: i64) {
+ // Check workspace_id before acquiring lock
+ let workspace_id = match self.workspace_id.load_full() {
+ Some(id) => id,
+ None => {
+ error!("No workspace id found");
+ return;
+ },
+ };
+
+ // Check and update current search
+ {
+ let mut current = self.current_search.lock().await;
+ if current.map_or(false, |cur| cur > search_id) {
+ return;
+ }
+ *current = Some(search_id);
+ }
+
+ info!("[Search] perform search: {}", query);
let handlers = self.handlers.clone();
let sink = IsolateSink::new(Isolate::new(stream_port));
- let mut join_handles = vec![];
let current_search = self.current_search.clone();
+ let mut join_handles = vec![];
- tracing::info!("[Search] perform search: {}", query);
- for (_, handler) in handlers {
- let mut clone_sink = sink.clone();
- let query = query.clone();
- let filter = filter.clone();
- let search_id = search_id.clone();
- let current_search = current_search.clone();
+ for handler in handlers.iter().map(|entry| entry.value().clone()) {
+ let mut sink_clone = sink.clone();
+ let query_clone = query.clone();
+ let current_search_clone = current_search.clone();
+ let workspace_id_clone = workspace_id.clone();
let handle = tokio::spawn(async move {
- if !is_current_search(&current_search, &search_id).await {
- trace!("[Search] cancel search: {}", query);
+ // Check if still current search before starting
+ if !is_current_search(&current_search_clone, search_id).await {
+ trace!("[Search] cancel search: {}", query_clone);
return;
}
- let mut stream = handler.perform_search(query.clone(), filter).await;
+ let mut stream = handler
+ .perform_search(query_clone.clone(), &workspace_id_clone)
+ .await;
+
while let Some(Ok(search_result)) = stream.next().await {
- if !is_current_search(&current_search, &search_id).await {
- trace!("[Search] discard search stream: {}", query);
+ if !is_current_search(&current_search_clone, search_id).await {
+ trace!("[Search] perform search cancel: {}", query_clone);
return;
}
let resp = SearchStatePB {
response: Some(search_result),
- search_id: search_id.clone(),
+ search_id: search_id.to_string(),
};
+
if let Ok::<Vec<u8>, _>(data) = resp.try_into() {
- if let Err(err) = clone_sink.send(data).await {
+ if let Err(err) = sink_clone.send(data).await {
error!("Failed to send search result: {}", err);
break;
}
}
}
- if !is_current_search(&current_search, &search_id).await {
- trace!("[Search] discard search result: {}", query);
- return;
- }
-
- let resp = SearchStatePB {
- response: None,
- search_id: search_id.clone(),
- };
- if let Ok::<Vec<u8>, _>(data) = resp.try_into() {
- let _ = clone_sink.send(data).await;
+ // Send completion message
+ if is_current_search(&current_search_clone, search_id).await {
+ let resp = SearchStatePB {
+ response: None,
+ search_id: search_id.to_string(),
+ };
+ if let Ok::<Vec<u8>, _>(data) = resp.try_into() {
+ let _ = sink_clone.send(data).await;
+ }
}
});
+
join_handles.push(handle);
}
+
futures::future::join_all(join_handles).await;
}
}
@@ -128,9 +196,12 @@ impl Drop for SearchManager {
}
async fn is_current_search(
- current_search: &Arc<tokio::sync::Mutex<Option<String>>>,
- search_id: &str,
+ current_search: &Arc<tokio::sync::Mutex<Option<i64>>>,
+ search_id: i64,
) -> bool {
let current = current_search.lock().await;
- current.as_ref().map_or(false, |id| id == search_id)
+ match *current {
+ None => true,
+ Some(c) => c == search_id,
+ }
}
diff --git a/frontend/rust-lib/flowy-search/tests/tantivy_test.rs b/frontend/rust-lib/flowy-search/tests/tantivy_test.rs
index f68b06d610..6506851d78 100644
--- a/frontend/rust-lib/flowy-search/tests/tantivy_test.rs
+++ b/frontend/rust-lib/flowy-search/tests/tantivy_test.rs
@@ -13,9 +13,8 @@ fn search_folder_test() {
// Indexing documents
let index = Index::create_from_tempdir(schema.clone()).unwrap();
- // Here we use a buffer of 100MB that will be split
// between indexing threads.
- let mut index_writer = index.writer(100_000_000).unwrap();
+ let mut index_writer = index.writer(15_000_000).unwrap();
// Let's index one documents!
index_writer
diff --git a/frontend/rust-lib/flowy-server/Cargo.toml b/frontend/rust-lib/flowy-server/Cargo.toml
index 65bb2e8dea..4edf6ee2e7 100644
--- a/frontend/rust-lib/flowy-server/Cargo.toml
+++ b/frontend/rust-lib/flowy-server/Cargo.toml
@@ -10,7 +10,6 @@ crate-type = ["cdylib", "rlib"]
[dependencies]
tracing.workspace = true
-futures.workspace = true
futures-util = "0.3.26"
serde.workspace = true
serde_json.workspace = true
@@ -25,7 +24,6 @@ collab = { workspace = true }
collab-plugins = { workspace = true }
collab-document = { workspace = true }
collab-entity = { workspace = true }
-collab-folder = { workspace = true }
collab-database = { workspace = true }
collab-user = { workspace = true }
lib-infra = { workspace = true }
@@ -62,5 +60,3 @@ features = [
uuid.workspace = true
serde_json.workspace = true
-[features]
-enable_supabase = ["collab-plugins/postgres_plugin"]
diff --git a/frontend/rust-lib/flowy-server/src/local_server/impls/mod.rs b/frontend/rust-lib/flowy-server/src/local_server/impls/mod.rs
index f63265e734..751de4e294 100644
--- a/frontend/rust-lib/flowy-server/src/local_server/impls/mod.rs
+++ b/frontend/rust-lib/flowy-server/src/local_server/impls/mod.rs
@@ -2,10 +2,12 @@ pub(crate) use chat::*;
pub(crate) use database::*;
pub(crate) use document::*;
pub(crate) use folder::*;
+pub(crate) use search::*;
pub(crate) use user::*;
mod chat;
mod database;
mod document;
mod folder;
+mod search;
mod user;
diff --git a/frontend/rust-lib/flowy-server/src/local_server/impls/search.rs b/frontend/rust-lib/flowy-server/src/local_server/impls/search.rs
new file mode 100644
index 0000000000..58820a8d82
--- /dev/null
+++ b/frontend/rust-lib/flowy-server/src/local_server/impls/search.rs
@@ -0,0 +1,65 @@
+use crate::af_cloud::define::LoggedUser;
+use flowy_ai::local_ai::controller::LocalAIController;
+use flowy_ai_pub::cloud::search_dto::{
+ SearchDocumentResponseItem, SearchResult, SearchSummaryResult,
+};
+use flowy_error::FlowyError;
+use flowy_search_pub::cloud::SearchCloudService;
+use lib_infra::async_trait::async_trait;
+use std::sync::Arc;
+use uuid::Uuid;
+
+pub struct LocalSearchServiceImpl {
+ #[allow(dead_code)]
+ pub logged_user: Arc<dyn LoggedUser>,
+ pub local_ai: Arc<LocalAIController>,
+}
+
+impl LocalSearchServiceImpl {}
+
+#[async_trait]
+impl SearchCloudService for LocalSearchServiceImpl {
+ async fn document_search(
+ &self,
+ workspace_id: &Uuid,
+ query: String,
+ ) -> Result<Vec<SearchDocumentResponseItem>, FlowyError> {
+ #[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))]
+ {
+ if let Ok(scheduler) = flowy_ai::embeddings::context::EmbedContext::shared().get_scheduler() {
+ match scheduler.search(workspace_id, &query).await {
+ Ok(results) => return Ok(results),
+ Err(err) => tracing::error!("Local AI search failed: {:?}", err),
+ }
+ } else {
+ tracing::error!("Could not acquire local AI scheduler");
+ }
+ }
+
+ Ok(Vec::new())
+ }
+
+ async fn generate_search_summary(
+ &self,
+ _workspace_id: &Uuid,
+ query: String,
+ search_results: Vec<SearchResult>,
+ ) -> Result<SearchSummaryResult, FlowyError> {
+ #[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))]
+ {
+ if let Ok(scheduler) = flowy_ai::embeddings::context::EmbedContext::shared().get_scheduler() {
+ let setting = self.local_ai.get_local_ai_setting();
+ match scheduler
+ .generate_summary(&query, &setting.chat_model_name, search_results)
+ .await
+ {
+ Ok(results) => return Ok(results),
+ Err(err) => tracing::error!("Local AI search failed: {:?}", err),
+ }
+ } else {
+ tracing::error!("Could not acquire local AI scheduler");
+ }
+ }
+ Ok(SearchSummaryResult { summaries: vec![] })
+ }
+}
diff --git a/frontend/rust-lib/flowy-server/src/local_server/server.rs b/frontend/rust-lib/flowy-server/src/local_server/server.rs
index 8829ded3fc..4e1103daf5 100644
--- a/frontend/rust-lib/flowy-server/src/local_server/server.rs
+++ b/frontend/rust-lib/flowy-server/src/local_server/server.rs
@@ -1,7 +1,8 @@
use crate::af_cloud::define::LoggedUser;
use crate::local_server::impls::{
- LocalChatServiceImpl, LocalServerDatabaseCloudServiceImpl, LocalServerDocumentCloudServiceImpl,
- LocalServerFolderCloudServiceImpl, LocalServerUserServiceImpl,
+ LocalChatServiceImpl, LocalSearchServiceImpl, LocalServerDatabaseCloudServiceImpl,
+ LocalServerDocumentCloudServiceImpl, LocalServerFolderCloudServiceImpl,
+ LocalServerUserServiceImpl,
};
use crate::AppFlowyServer;
use anyhow::Error;
@@ -78,7 +79,10 @@ impl AppFlowyServer for LocalServer {
}
fn search_service(&self) -> Option<Arc<dyn SearchCloudService>> {
- None
+ Some(Arc::new(LocalSearchServiceImpl {
+ logged_user: self.logged_user.clone(),
+ local_ai: self.local_ai.clone(),
+ }))
}
fn file_storage(&self) -> Option<Arc<dyn StorageCloudService>> {
diff --git a/frontend/rust-lib/flowy-server/src/server.rs b/frontend/rust-lib/flowy-server/src/server.rs
index 2702b4f104..ed65971c95 100644
--- a/frontend/rust-lib/flowy-server/src/server.rs
+++ b/frontend/rust-lib/flowy-server/src/server.rs
@@ -9,8 +9,6 @@ use arc_swap::ArcSwapOption;
use client_api::collab_sync::ServerCollabMessage;
use flowy_ai_pub::cloud::ChatCloudService;
use tokio_stream::wrappers::WatchStream;
-#[cfg(feature = "enable_supabase")]
-use {collab_entity::CollabObject, collab_plugins::cloud_storage::RemoteCollabStorage};
use flowy_database_pub::cloud::{DatabaseAIService, DatabaseCloudService};
use flowy_document_pub::cloud::DocumentCloudService;
@@ -106,22 +104,6 @@ pub trait AppFlowyServer: Send + Sync + 'static {
///
fn search_service(&self) -> Option<Arc<dyn SearchCloudService>>;
- /// Manages collaborative objects within a remote storage system. This includes operations such as
- /// checking storage status, retrieving updates and snapshots, and dispatching updates. The service
- /// also provides subscription capabilities for real-time updates.
- ///
- /// # Arguments
- ///
- /// * `collab_object` - A reference to the collaborative object.
- ///
- /// # Returns
- ///
- /// An `Option` that might contain an `Arc` wrapping the `RemoteCollabStorage` interface.
- #[cfg(feature = "enable_supabase")]
- fn collab_storage(&self, _collab_object: &CollabObject) -> Option<Arc<dyn RemoteCollabStorage>> {
- None
- }
-
fn subscribe_ws_state(&self) -> Option<WSConnectStateReceiver> {
None
}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/api/collab_storage.rs b/frontend/rust-lib/flowy-server/src/supabase/api/collab_storage.rs
deleted file mode 100644
index bb5705cbc8..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/api/collab_storage.rs
+++ /dev/null
@@ -1,295 +0,0 @@
-use std::str::FromStr;
-use std::sync::{Arc, Weak};
-
-use anyhow::Error;
-use arc_swap::ArcSwapOption;
-use chrono::{DateTime, Utc};
-use client_api::collab_sync::MsgId;
-use collab::core::collab::DataSource;
-use collab::preclude::merge_updates_v1;
-use collab_entity::CollabObject;
-use collab_plugins::cloud_storage::{
- RemoteCollabSnapshot, RemoteCollabState, RemoteCollabStorage, RemoteUpdateReceiver,
-};
-use tokio::task::spawn_blocking;
-
-use lib_infra::async_trait::async_trait;
-use lib_infra::util::md5;
-
-use crate::response::ExtendedResponse;
-use crate::supabase::api::request::{
- create_snapshot, get_snapshots_from_server, get_updates_from_server, FetchObjectUpdateAction,
- UpdateItem,
-};
-use crate::supabase::api::util::{InsertParamsBuilder, SupabaseBinaryColumnEncoder};
-use crate::supabase::api::{PostgresWrapper, SupabaseServerService};
-use crate::supabase::define::*;
-use crate::AppFlowyEncryption;
-
-pub struct SupabaseCollabStorageImpl<T> {
- server: T,
- rx: ArcSwapOption<RemoteUpdateReceiver>,
- encryption: Weak<dyn AppFlowyEncryption>,
-}
-
-impl<T> SupabaseCollabStorageImpl<T> {
- pub fn new(
- server: T,
- rx: Option<RemoteUpdateReceiver>,
- encryption: Weak<dyn AppFlowyEncryption>,
- ) -> Self {
- Self {
- server,
- rx: ArcSwapOption::new(rx.map(Arc::new)),
- encryption,
- }
- }
-
- pub fn secret(&self) -> Option<String> {
- match self.encryption.upgrade() {
- None => None,
- Some(encryption) => encryption.get_secret(),
- }
- }
-}
-
-#[async_trait]
-impl<T> RemoteCollabStorage for SupabaseCollabStorageImpl<T>
-where
- T: SupabaseServerService,
-{
- fn is_enable(&self) -> bool {
- true
- }
-
- async fn get_doc_state(&self, object: &CollabObject) -> Result<DataSource, Error> {
- let postgrest = self.server.try_get_weak_postgrest()?;
- let action = FetchObjectUpdateAction::new(
- object.object_id.clone(),
- object.collab_type.clone(),
- postgrest,
- );
- let doc_state = action.run().await?;
- Ok(DataSource::DocStateV1(doc_state))
- }
-
- async fn get_snapshots(&self, object_id: &str, limit: usize) -> Vec<RemoteCollabSnapshot> {
- match self.server.try_get_postgrest() {
- Ok(postgrest) => get_snapshots_from_server(object_id, postgrest, limit)
- .await
- .unwrap_or_else(|err| {
- tracing::error!(
- "🔴fetch snapshots by oid:{} with limit: {} failed: {:?}",
- object_id,
- limit,
- err
- );
- vec![]
- }),
- Err(err) => {
- tracing::error!("🔴get postgrest failed: {:?}", err);
- vec![]
- },
- }
- }
-
- async fn get_collab_state(&self, object_id: &str) -> Result<Option<RemoteCollabState>, Error> {
- let postgrest = self.server.try_get_postgrest()?;
- let json = postgrest
- .from("af_collab_state")
- .select("*")
- .eq("oid", object_id)
- .order("snapshot_created_at.desc".to_string())
- .limit(1)
- .execute()
- .await?
- .get_json()
- .await?;
-
- Ok(
- json
- .as_array()
- .and_then(|array| array.first())
- .and_then(|value| {
- let created_at = value.get("snapshot_created_at").and_then(|created_at| {
- created_at
- .as_str()
- .map(|id| DateTime::<Utc>::from_str(id).ok())
- .and_then(|date| date)
- })?;
-
- let current_edit_count = value.get("current_edit_count").and_then(|id| id.as_i64())?;
- let snapshot_edit_count = value
- .get("snapshot_edit_count")
- .and_then(|id| id.as_i64())?;
-
- Some(RemoteCollabState {
- current_edit_count,
- snapshot_edit_count,
- snapshot_created_at: created_at.timestamp(),
- })
- }),
- )
- }
-
- async fn create_snapshot(&self, object: &CollabObject, snapshot: Vec<u8>) -> Result<i64, Error> {
- let postgrest = self.server.try_get_postgrest()?;
- create_snapshot(&postgrest, object, snapshot).await
- }
-
- async fn send_update(
- &self,
- object: &CollabObject,
- _id: MsgId,
- update: Vec<u8>,
- ) -> Result<(), Error> {
- if let Some(postgrest) = self.server.get_postgrest() {
- send_update(
- object.workspace_id.clone(),
- object,
- update,
- &postgrest,
- &self.secret(),
- )
- .await?;
- }
-
- Ok(())
- }
-
- async fn send_init_sync(
- &self,
- object: &CollabObject,
- _id: MsgId,
- init_update: Vec<u8>,
- ) -> Result<(), Error> {
- let postgrest = self.server.try_get_postgrest()?;
-
- let update_items =
- get_updates_from_server(&object.object_id, &object.collab_type, &postgrest).await?;
-
- // If the update_items is empty, we can send the init_update directly
- if update_items.is_empty() {
- send_update(
- object.workspace_id.clone(),
- object,
- init_update,
- &postgrest,
- &self.secret(),
- )
- .await?;
- } else {
- flush_collab_with_update(object, update_items, &postgrest, init_update, self.secret())
- .await?;
- }
- Ok(())
- }
-
- fn subscribe_remote_updates(&self, _object: &CollabObject) -> Option<RemoteUpdateReceiver> {
- let rx = self.rx.swap(None);
- match rx {
- Some(rx) => Arc::into_inner(rx),
- None => {
- tracing::warn!("The receiver is already taken");
- None
- },
- }
- }
-}
-
-pub(crate) async fn flush_collab_with_update(
- object: &CollabObject,
- update_items: Vec<UpdateItem>,
- postgrest: &Arc<PostgresWrapper>,
- update: Vec<u8>,
- secret: Option<String>,
-) -> Result<(), Error> {
- // 2.Merge the updates into one and then delete the merged updates
- let merge_result = spawn_blocking(move || merge_updates(update_items, update)).await??;
- let value_size = merge_result.new_update.len() as i32;
- let md5 = md5(&merge_result.new_update);
-
- tracing::trace!(
- "Flush collab id:{} type:{} is_encrypt: {}",
- object.object_id,
- object.collab_type,
- secret.is_some()
- );
- let (new_update, encrypt) =
- SupabaseBinaryColumnEncoder::encode(merge_result.new_update, &secret)?;
- let params = InsertParamsBuilder::new()
- .insert("oid", object.object_id.clone())
- .insert("new_value", new_update)
- .insert("encrypt", encrypt)
- .insert("md5", md5)
- .insert("value_size", value_size)
- .insert("partition_key", partition_key(&object.collab_type))
- .insert("uid", object.uid)
- .insert("workspace_id", &object.workspace_id)
- .insert("removed_keys", merge_result.merged_keys)
- .insert("did", &object.device_id)
- .build();
-
- postgrest
- .rpc("flush_collab_updates_v3", params)
- .execute()
- .await?
- .success()
- .await?;
- Ok(())
-}
-
-pub(crate) async fn send_update(
- workspace_id: String,
- object: &CollabObject,
- update: Vec<u8>,
- postgrest: &Arc<PostgresWrapper>,
- encryption_secret: &Option<String>,
-) -> Result<(), Error> {
- let value_size = update.len() as i32;
- let md5 = md5(&update);
- let (update, encrypt) = SupabaseBinaryColumnEncoder::encode(update, encryption_secret)?;
- let builder = InsertParamsBuilder::new()
- .insert("oid", object.object_id.clone())
- .insert("partition_key", partition_key(&object.collab_type))
- .insert("value", update)
- .insert("encrypt", encrypt)
- .insert("uid", object.uid)
- .insert("md5", md5)
- .insert("workspace_id", workspace_id)
- .insert("did", &object.device_id)
- .insert("value_size", value_size);
-
- let params = builder.build();
- postgrest
- .from(AF_COLLAB_UPDATE_TABLE)
- .insert(params)
- .execute()
- .await?
- .success()
- .await?;
- Ok(())
-}
-
-fn merge_updates(update_items: Vec<UpdateItem>, new_update: Vec<u8>) -> Result<MergeResult, Error> {
- let mut updates = vec![];
- let mut merged_keys = vec![];
- for item in update_items {
- merged_keys.push(item.key);
- updates.push(item.value);
- }
- if !new_update.is_empty() {
- updates.push(new_update);
- }
-
- let new_update = merge_updates_v1(updates)?;
- Ok(MergeResult {
- merged_keys,
- new_update,
- })
-}
-
-struct MergeResult {
- merged_keys: Vec<i64>,
- new_update: Vec<u8>,
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/api/database.rs b/frontend/rust-lib/flowy-server/src/supabase/api/database.rs
deleted file mode 100644
index af1732b500..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/api/database.rs
+++ /dev/null
@@ -1,99 +0,0 @@
-use anyhow::Error;
-use collab_entity::CollabType;
-use tokio::sync::oneshot::channel;
-
-use flowy_database_pub::cloud::{CollabDocStateByOid, DatabaseCloudService, DatabaseSnapshot};
-
-use lib_infra::async_trait::async_trait;
-use lib_infra::future::FutureResult;
-
-use crate::supabase::api::request::{
- get_snapshots_from_server, BatchFetchObjectUpdateAction, FetchObjectUpdateAction,
-};
-use crate::supabase::api::SupabaseServerService;
-
-pub struct SupabaseDatabaseServiceImpl<T> {
- server: T,
-}
-
-impl<T> SupabaseDatabaseServiceImpl<T> {
- pub fn new(server: T) -> Self {
- Self { server }
- }
-}
-
-#[async_trait]
-impl<T> DatabaseCloudService for SupabaseDatabaseServiceImpl<T>
-where
- T: SupabaseServerService,
-{
- fn get_database_object_doc_state(
- &self,
- object_id: &str,
- collab_type: CollabType,
- _workspace_id: &str,
- ) -> FutureResult<Option<Vec<u8>>, Error> {
- let try_get_postgrest = self.server.try_get_weak_postgrest();
- let object_id = object_id.to_string();
- let (tx, rx) = channel();
- tokio::spawn(async move {
- tx.send(
- async move {
- let postgrest = try_get_postgrest?;
- let updates = FetchObjectUpdateAction::new(object_id.to_string(), collab_type, postgrest)
- .run_with_fix_interval(5, 10)
- .await?;
- Ok(Some(updates))
- }
- .await,
- )
- });
- FutureResult::new(async { rx.await? })
- }
-
- fn batch_get_database_object_doc_state(
- &self,
- object_ids: Vec<String>,
- object_ty: CollabType,
- _workspace_id: &str,
- ) -> FutureResult<CollabDocStateByOid, Error> {
- let try_get_postgrest = self.server.try_get_weak_postgrest();
- let (tx, rx) = channel();
- tokio::spawn(async move {
- tx.send(
- async move {
- let postgrest = try_get_postgrest?;
- BatchFetchObjectUpdateAction::new(object_ids, object_ty, postgrest)
- .run()
- .await
- }
- .await,
- )
- });
- FutureResult::new(async { rx.await? })
- }
-
- fn get_database_collab_object_snapshots(
- &self,
- object_id: &str,
- limit: usize,
- ) -> FutureResult<Vec<DatabaseSnapshot>, Error> {
- let try_get_postgrest = self.server.try_get_postgrest();
- let object_id = object_id.to_string();
- FutureResult::new(async move {
- let postgrest = try_get_postgrest?;
- let snapshots = get_snapshots_from_server(&object_id, postgrest, limit)
- .await?
- .into_iter()
- .map(|snapshot| DatabaseSnapshot {
- snapshot_id: snapshot.sid,
- database_id: snapshot.oid,
- data: snapshot.blob,
- created_at: snapshot.created_at,
- })
- .collect::<Vec<_>>();
-
- Ok(snapshots)
- })
- }
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/api/document.rs b/frontend/rust-lib/flowy-server/src/supabase/api/document.rs
deleted file mode 100644
index b3f45a7670..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/api/document.rs
+++ /dev/null
@@ -1,110 +0,0 @@
-use anyhow::Error;
-use collab::core::collab::DataSource;
-use collab::core::origin::CollabOrigin;
-use collab_document::blocks::DocumentData;
-use collab_document::document::Document;
-use collab_entity::CollabType;
-use tokio::sync::oneshot::channel;
-
-use flowy_document_pub::cloud::{DocumentCloudService, DocumentSnapshot};
-use flowy_error::FlowyError;
-
-use lib_infra::future::FutureResult;
-
-use crate::supabase::api::request::{get_snapshots_from_server, FetchObjectUpdateAction};
-use crate::supabase::api::SupabaseServerService;
-
-pub struct SupabaseDocumentServiceImpl<T> {
- server: T,
-}
-
-impl<T> SupabaseDocumentServiceImpl<T> {
- pub fn new(server: T) -> Self {
- Self { server }
- }
-}
-
-impl<T> DocumentCloudService for SupabaseDocumentServiceImpl<T>
-where
- T: SupabaseServerService,
-{
- #[tracing::instrument(level = "debug", skip(self))]
- fn get_document_doc_state(
- &self,
- document_id: &str,
- workspace_id: &str,
- ) -> FutureResult<Vec<u8>, FlowyError> {
- let try_get_postgrest = self.server.try_get_weak_postgrest();
- let document_id = document_id.to_string();
- let (tx, rx) = channel();
- tokio::spawn(async move {
- tx.send(
- async move {
- let postgrest = try_get_postgrest?;
- let action = FetchObjectUpdateAction::new(document_id, CollabType::Document, postgrest);
- let collab_doc_state = action.run_with_fix_interval(5, 10).await?;
- if collab_doc_state.is_empty() {
- return Err(FlowyError::collab_not_sync());
- }
- Ok(collab_doc_state)
- }
- .await,
- )
- });
- FutureResult::new(async { rx.await? })
- }
-
- fn get_document_snapshots(
- &self,
- document_id: &str,
- limit: usize,
- _workspace_id: &str,
- ) -> FutureResult<Vec<DocumentSnapshot>, Error> {
- let try_get_postgrest = self.server.try_get_postgrest();
- let document_id = document_id.to_string();
- FutureResult::new(async move {
- let postgrest = try_get_postgrest?;
- let snapshots = get_snapshots_from_server(&document_id, postgrest, limit)
- .await?
- .into_iter()
- .map(|snapshot| DocumentSnapshot {
- snapshot_id: snapshot.sid,
- document_id: snapshot.oid,
- data: snapshot.blob,
- created_at: snapshot.created_at,
- })
- .collect::<Vec<_>>();
- Ok(snapshots)
- })
- }
-
- #[tracing::instrument(level = "debug", skip(self))]
- fn get_document_data(
- &self,
- document_id: &str,
- _workspace_id: &str,
- ) -> FutureResult<Option<DocumentData>, Error> {
- let try_get_postgrest = self.server.try_get_weak_postgrest();
- let document_id = document_id.to_string();
- let (tx, rx) = channel();
- tokio::spawn(async move {
- tx.send(
- async move {
- let postgrest = try_get_postgrest?;
- let action =
- FetchObjectUpdateAction::new(document_id.clone(), CollabType::Document, postgrest);
- let doc_state = action.run_with_fix_interval(5, 10).await?;
- let document = Document::open_with_options(
- CollabOrigin::Empty,
- DataSource::DocStateV1(doc_state),
- &document_id,
- vec![],
- )?;
- Ok(document.get_document_data().ok())
- }
- .await,
- )
- });
- FutureResult::new(async { rx.await? })
- }
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/api/folder.rs b/frontend/rust-lib/flowy-server/src/supabase/api/folder.rs
deleted file mode 100644
index 253d11e0d8..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/api/folder.rs
+++ /dev/null
@@ -1,245 +0,0 @@
-use std::str::FromStr;
-
-use anyhow::{anyhow, Error};
-use chrono::{DateTime, Utc};
-use collab::core::collab::DataSource;
-use collab::core::origin::CollabOrigin;
-use collab_entity::CollabType;
-use serde_json::Value;
-use tokio::sync::oneshot::channel;
-use yrs::merge_updates_v1;
-
-use flowy_folder_pub::cloud::{
- gen_workspace_id, Folder, FolderCloudService, FolderCollabParams, FolderData, FolderSnapshot,
- Workspace, WorkspaceRecord,
-};
-use flowy_folder_pub::entities::PublishPayload;
-
-use lib_infra::future::FutureResult;
-use lib_infra::util::timestamp;
-
-use crate::response::ExtendedResponse;
-use crate::supabase::api::request::{
- get_snapshots_from_server, get_updates_from_server, FetchObjectUpdateAction,
-};
-use crate::supabase::api::util::InsertParamsBuilder;
-use crate::supabase::api::SupabaseServerService;
-use crate::supabase::define::*;
-
-pub struct SupabaseFolderServiceImpl<T> {
- server: T,
-}
-
-impl<T> SupabaseFolderServiceImpl<T> {
- pub fn new(server: T) -> Self {
- Self { server }
- }
-}
-
-impl<T> FolderCloudService for SupabaseFolderServiceImpl<T>
-where
- T: SupabaseServerService,
-{
- fn create_workspace(&self, uid: i64, name: &str) -> FutureResult<Workspace, Error> {
- let try_get_postgrest = self.server.try_get_postgrest();
- let name = name.to_string();
- let new_workspace_id = gen_workspace_id().to_string();
- FutureResult::new(async move {
- let postgrest = try_get_postgrest?;
- let insert_params = InsertParamsBuilder::new()
- .insert(OWNER_USER_UID, uid)
- .insert(WORKSPACE_ID, new_workspace_id.clone())
- .insert(WORKSPACE_NAME, name.to_string())
- .build();
- postgrest
- .from(WORKSPACE_TABLE)
- .insert(insert_params)
- .execute()
- .await?
- .success()
- .await?;
-
- // read the workspace
- let json = postgrest
- .from(WORKSPACE_TABLE)
- .select("*")
- .eq(WORKSPACE_ID, new_workspace_id)
- .execute()
- .await?
- .get_json()
- .await?;
-
- let workspace = workspace_from_json_value(json)?;
- Ok(workspace)
- })
- }
-
- fn open_workspace(&self, _workspace_id: &str) -> FutureResult<(), Error> {
- FutureResult::new(async { Ok(()) })
- }
-
- fn get_all_workspace(&self) -> FutureResult<Vec<WorkspaceRecord>, Error> {
- FutureResult::new(async { Ok(vec![]) })
- }
-
- fn get_folder_data(
- &self,
- workspace_id: &str,
- uid: &i64,
- ) -> FutureResult<Option<FolderData>, Error> {
- let uid = *uid;
- let try_get_postgrest = self.server.try_get_postgrest();
- let workspace_id = workspace_id.to_string();
- FutureResult::new(async move {
- let postgrest = try_get_postgrest?;
- let items = get_updates_from_server(&workspace_id, &CollabType::Folder, &postgrest).await?;
- if items.is_empty() {
- return Ok(None);
- }
- let updates = items.into_iter().map(|update| update.value);
- let doc_state = merge_updates_v1(updates)
- .map_err(|err| anyhow::anyhow!("merge updates failed: {:?}", err))?;
-
- let folder = Folder::from_collab_doc_state(
- uid,
- CollabOrigin::Empty,
- DataSource::DocStateV1(doc_state),
- &workspace_id,
- vec![],
- )?;
- Ok(folder.get_folder_data(&workspace_id))
- })
- }
-
- fn get_folder_snapshots(
- &self,
- workspace_id: &str,
- limit: usize,
- ) -> FutureResult<Vec<FolderSnapshot>, Error> {
- let try_get_postgrest = self.server.try_get_postgrest();
- let workspace_id = workspace_id.to_string();
- FutureResult::new(async move {
- let postgrest = try_get_postgrest?;
- let snapshots = get_snapshots_from_server(&workspace_id, postgrest, limit)
- .await?
- .into_iter()
- .map(|snapshot| FolderSnapshot {
- snapshot_id: snapshot.sid,
- database_id: snapshot.oid,
- data: snapshot.blob,
- created_at: snapshot.created_at,
- })
- .collect::<Vec<_>>();
- Ok(snapshots)
- })
- }
-
- fn get_folder_doc_state(
- &self,
- _workspace_id: &str,
- _uid: i64,
- collab_type: CollabType,
- object_id: &str,
- ) -> FutureResult<Vec<u8>, Error> {
- let try_get_postgrest = self.server.try_get_weak_postgrest();
- let object_id = object_id.to_string();
- let (tx, rx) = channel();
- tokio::spawn(async move {
- tx.send(
- async move {
- let postgrest = try_get_postgrest?;
- let action = FetchObjectUpdateAction::new(object_id, collab_type, postgrest);
- action.run_with_fix_interval(5, 10).await
- }
- .await,
- )
- });
- FutureResult::new(async { rx.await? })
- }
-
- fn batch_create_folder_collab_objects(
- &self,
- _workspace_id: &str,
- _objects: Vec<FolderCollabParams>,
- ) -> FutureResult<(), Error> {
- FutureResult::new(async {
- Err(anyhow!(
- "supabase server doesn't support batch create collab"
- ))
- })
- }
-
- fn service_name(&self) -> String {
- "Supabase".to_string()
- }
-
- fn publish_view(
- &self,
- _workspace_id: &str,
- _payload: Vec<PublishPayload>,
- ) -> FutureResult<(), Error> {
- FutureResult::new(async { Err(anyhow!("supabase server doesn't support publish view")) })
- }
-
- fn unpublish_views(
- &self,
- _workspace_id: &str,
- _view_ids: Vec<String>,
- ) -> FutureResult<(), Error> {
- FutureResult::new(async { Err(anyhow!("supabase server doesn't support unpublish views")) })
- }
-
- fn get_publish_info(&self, _view_id: &str) -> FutureResult<PublishInfo, Error> {
- FutureResult::new(async { Err(anyhow!("supabase server doesn't support publish info")) })
- }
-
- fn set_publish_namespace(
- &self,
- _workspace_id: &str,
- _new_namespace: &str,
- ) -> FutureResult<(), FlowyError> {
- FutureResult::new(async {
- Err(anyhow!(
- "supabase server doesn't support set publish namespace"
- ))
- })
- }
-
- fn get_publish_namespace(&self, _workspace_id: &str) -> FutureResult<String, Error> {
- FutureResult::new(async {
- Err(anyhow!(
- "supabase server doesn't support get publish namespace"
- ))
- })
- }
-}
-
-fn workspace_from_json_value(value: Value) -> Result<Workspace, Error> {
- let json = value
- .as_array()
- .and_then(|values| values.first())
- .ok_or(anyhow::anyhow!("workspace not found"))?;
- Ok(Workspace {
- id: json
- .get(WORKSPACE_ID)
- .ok_or(anyhow::anyhow!("workspace id not found"))?
- .to_string(),
- name: json
- .get(WORKSPACE_NAME)
- .map(|value| value.to_string())
- .unwrap_or_default(),
- child_views: Default::default(),
- created_at: json
- .get(CREATED_AT)
- .and_then(|value| value.as_str())
- .and_then(|s| DateTime::<Utc>::from_str(s).ok())
- .map(|date| date.timestamp())
- .unwrap_or_default(),
- created_by: json.get("created_by").and_then(|value| value.as_i64()),
- last_edited_time: json
- .get("last_edited_time")
- .and_then(|value| value.as_i64())
- .unwrap_or(timestamp()),
- last_edited_by: json.get("last_edited_by").and_then(|value| value.as_i64()),
- })
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/api/mod.rs b/frontend/rust-lib/flowy-server/src/supabase/api/mod.rs
deleted file mode 100644
index 6134109c1f..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/api/mod.rs
+++ /dev/null
@@ -1,15 +0,0 @@
-pub use collab_storage::*;
-pub use database::*;
-pub use document::*;
-pub use folder::*;
-pub use postgres_server::*;
-pub use user::*;
-
-mod collab_storage;
-mod database;
-mod document;
-mod folder;
-mod postgres_server;
-mod request;
-mod user;
-pub mod util;
diff --git a/frontend/rust-lib/flowy-server/src/supabase/api/postgres_server.rs b/frontend/rust-lib/flowy-server/src/supabase/api/postgres_server.rs
deleted file mode 100644
index 9ab3379486..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/api/postgres_server.rs
+++ /dev/null
@@ -1,116 +0,0 @@
-use std::ops::Deref;
-use std::sync::{Arc, Weak};
-
-use anyhow::Error;
-use arc_swap::ArcSwapOption;
-use postgrest::Postgrest;
-
-use flowy_error::{ErrorCode, FlowyError};
-use flowy_server_pub::supabase_config::SupabaseConfiguration;
-
-use crate::AppFlowyEncryption;
-
-/// Creates a wrapper for Postgrest, which allows us to extend the functionality of Postgrest.
-pub struct PostgresWrapper {
- inner: Postgrest,
- pub encryption: Weak<dyn AppFlowyEncryption>,
-}
-
-impl PostgresWrapper {
- pub fn secret(&self) -> Option<String> {
- match self.encryption.upgrade() {
- None => None,
- Some(encryption) => encryption.get_secret(),
- }
- }
-}
-
-impl Deref for PostgresWrapper {
- type Target = Postgrest;
-
- fn deref(&self) -> &Self::Target {
- &self.inner
- }
-}
-
-pub struct RESTfulPostgresServer {
- pub postgrest: Arc<PostgresWrapper>,
-}
-
-impl RESTfulPostgresServer {
- pub fn new(config: SupabaseConfiguration, encryption: Weak<dyn AppFlowyEncryption>) -> Self {
- let url = format!("{}/rest/v1", config.url);
- let auth = format!("Bearer {}", config.anon_key);
- let postgrest = Postgrest::new(url)
- .insert_header("apikey", config.anon_key)
- .insert_header("Authorization", auth);
- Self {
- postgrest: Arc::new(PostgresWrapper {
- inner: postgrest,
- encryption,
- }),
- }
- }
-}
-
-pub trait SupabaseServerService: Send + Sync + 'static {
- fn get_postgrest(&self) -> Option<Arc<PostgresWrapper>>;
- fn try_get_postgrest(&self) -> Result<Arc<PostgresWrapper>, Error>;
- fn try_get_weak_postgrest(&self) -> Result<Weak<PostgresWrapper>, Error>;
-}
-
-impl<T> SupabaseServerService for Arc<T>
-where
- T: SupabaseServerService,
-{
- fn get_postgrest(&self) -> Option<Arc<PostgresWrapper>> {
- (**self).get_postgrest()
- }
-
- fn try_get_postgrest(&self) -> Result<Arc<PostgresWrapper>, Error> {
- (**self).try_get_postgrest()
- }
-
- fn try_get_weak_postgrest(&self) -> Result<Weak<PostgresWrapper>, Error> {
- (**self).try_get_weak_postgrest()
- }
-}
-
-#[derive(Clone)]
-pub struct SupabaseServerServiceImpl(pub Arc<ArcSwapOption<RESTfulPostgresServer>>);
-
-impl SupabaseServerServiceImpl {
- pub fn new(postgrest: Arc<RESTfulPostgresServer>) -> Self {
- Self(Arc::new(ArcSwapOption::from(Some(postgrest))))
- }
-}
-
-impl SupabaseServerService for SupabaseServerServiceImpl {
- fn get_postgrest(&self) -> Option<Arc<PostgresWrapper>> {
- self
- .0
- .load()
- .as_ref()
- .map(|server| server.postgrest.clone())
- }
-
- fn try_get_postgrest(&self) -> Result<Arc<PostgresWrapper>, Error> {
- self
- .0
- .load()
- .as_ref()
- .map(|server| server.postgrest.clone())
- .ok_or_else(|| {
- FlowyError::new(
- ErrorCode::DataSyncRequired,
- "Data Sync is disabled, please enable it first",
- )
- .into()
- })
- }
-
- fn try_get_weak_postgrest(&self) -> Result<Weak<PostgresWrapper>, Error> {
- let postgrest = self.try_get_postgrest()?;
- Ok(Arc::downgrade(&postgrest))
- }
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/api/request.rs b/frontend/rust-lib/flowy-server/src/supabase/api/request.rs
deleted file mode 100644
index fa13c9711b..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/api/request.rs
+++ /dev/null
@@ -1,439 +0,0 @@
-use std::future::Future;
-use std::iter::Take;
-use std::pin::Pin;
-use std::str::FromStr;
-use std::sync::{Arc, Weak};
-use std::time::Duration;
-
-use anyhow::Error;
-use chrono::{DateTime, Utc};
-use collab::core::collab::DataSource;
-use collab_entity::{CollabObject, CollabType};
-use collab_plugins::cloud_storage::RemoteCollabSnapshot;
-use serde_json::Value;
-use tokio_retry::strategy::FixedInterval;
-use tokio_retry::{Action, Condition, RetryIf};
-use yrs::merge_updates_v1;
-
-use flowy_database_pub::cloud::CollabDocStateByOid;
-use lib_infra::util::md5;
-
-use crate::response::ExtendedResponse;
-use crate::supabase::api::util::{
- BinaryColumnDecoder, InsertParamsBuilder, SupabaseBinaryColumnDecoder,
- SupabaseBinaryColumnEncoder,
-};
-use crate::supabase::api::PostgresWrapper;
-use crate::supabase::define::*;
-
-pub struct FetchObjectUpdateAction {
- object_id: String,
- object_ty: CollabType,
- postgrest: Weak<PostgresWrapper>,
-}
-
-impl FetchObjectUpdateAction {
- pub fn new(object_id: String, object_ty: CollabType, postgrest: Weak<PostgresWrapper>) -> Self {
- Self {
- postgrest,
- object_id,
- object_ty,
- }
- }
-
- pub fn run(self) -> RetryIf<Take<FixedInterval>, FetchObjectUpdateAction, RetryCondition> {
- let postgrest = self.postgrest.clone();
- let retry_strategy = FixedInterval::new(Duration::from_secs(5)).take(3);
- RetryIf::spawn(retry_strategy, self, RetryCondition(postgrest))
- }
-
- pub fn run_with_fix_interval(
- self,
- secs: u64,
- times: usize,
- ) -> RetryIf<Take<FixedInterval>, FetchObjectUpdateAction, RetryCondition> {
- let postgrest = self.postgrest.clone();
- let retry_strategy = FixedInterval::new(Duration::from_secs(secs)).take(times);
- RetryIf::spawn(retry_strategy, self, RetryCondition(postgrest))
- }
-}
-
-impl Action for FetchObjectUpdateAction {
- type Future = Pin<Box<dyn Future<Output = Result<Self::Item, Self::Error>> + Send>>;
- type Item = Vec<u8>;
- type Error = anyhow::Error;
-
- fn run(&mut self) -> Self::Future {
- let weak_postgres = self.postgrest.clone();
- let object_id = self.object_id.clone();
- let object_ty = self.object_ty.clone();
- Box::pin(async move {
- match weak_postgres.upgrade() {
- None => Ok(vec![]),
- Some(postgrest) => {
- match get_updates_from_server(&object_id, &object_ty, &postgrest).await {
- Ok(items) => {
- if items.is_empty() {
- return Ok(vec![]);
- }
-
- let updates = items.into_iter().map(|update| update.value);
- let doc_state = merge_updates_v1(updates)
- .map_err(|err| anyhow::anyhow!("merge updates failed: {:?}", err))?;
- Ok(doc_state)
- },
- Err(err) => {
- tracing::error!("Get {} updates failed with error: {:?}", object_id, err);
- Err(err)
- },
- }
- },
- }
- })
- }
-}
-
-pub struct BatchFetchObjectUpdateAction {
- object_ids: Vec<String>,
- object_ty: CollabType,
- postgrest: Weak<PostgresWrapper>,
-}
-
-impl BatchFetchObjectUpdateAction {
- pub fn new(
- object_ids: Vec<String>,
- object_ty: CollabType,
- postgrest: Weak<PostgresWrapper>,
- ) -> Self {
- Self {
- postgrest,
- object_ty,
- object_ids,
- }
- }
-
- pub fn run(self) -> RetryIf<Take<FixedInterval>, BatchFetchObjectUpdateAction, RetryCondition> {
- let postgrest = self.postgrest.clone();
- let retry_strategy = FixedInterval::new(Duration::from_secs(5)).take(3);
- RetryIf::spawn(retry_strategy, self, RetryCondition(postgrest))
- }
-}
-
-impl Action for BatchFetchObjectUpdateAction {
- type Future = Pin<Box<dyn Future<Output = Result<Self::Item, Self::Error>> + Send>>;
- type Item = CollabDocStateByOid;
- type Error = anyhow::Error;
-
- fn run(&mut self) -> Self::Future {
- let weak_postgrest = self.postgrest.clone();
- let object_ids = self.object_ids.clone();
- let object_ty = self.object_ty.clone();
- Box::pin(async move {
- match weak_postgrest.upgrade() {
- None => Ok(CollabDocStateByOid::default()),
- Some(server) => {
- match batch_get_updates_from_server(object_ids.clone(), &object_ty, server).await {
- Ok(updates_by_oid) => Ok(updates_by_oid),
- Err(err) => {
- tracing::error!(
- "Batch get object with given ids:{:?} failed with error: {:?}",
- object_ids,
- err
- );
- Err(err)
- },
- }
- },
- }
- })
- }
-}
-
-pub async fn create_snapshot(
- postgrest: &Arc<PostgresWrapper>,
- object: &CollabObject,
- snapshot: Vec<u8>,
-) -> Result<i64, Error> {
- let value_size = snapshot.len() as i32;
- let (snapshot, encrypt) = SupabaseBinaryColumnEncoder::encode(&snapshot, &postgrest.secret())?;
- let ret: Value = postgrest
- .from(AF_COLLAB_SNAPSHOT_TABLE)
- .insert(
- InsertParamsBuilder::new()
- .insert(AF_COLLAB_SNAPSHOT_OID_COLUMN, object.object_id.clone())
- .insert("name", object.collab_type.to_string())
- .insert(AF_COLLAB_SNAPSHOT_ENCRYPT_COLUMN, encrypt)
- .insert(AF_COLLAB_SNAPSHOT_BLOB_COLUMN, snapshot)
- .insert(AF_COLLAB_SNAPSHOT_BLOB_SIZE_COLUMN, value_size)
- .build(),
- )
- .execute()
- .await?
- .get_json()
- .await?;
-
- let snapshot_id = ret
- .as_array()
- .and_then(|array| array.first())
- .and_then(|value| value.get("sid"))
- .and_then(|value| value.as_i64())
- .unwrap_or(0);
- Ok(snapshot_id)
-}
-
-pub async fn get_snapshots_from_server(
- object_id: &str,
- postgrest: Arc<PostgresWrapper>,
- limit: usize,
-) -> Result<Vec<RemoteCollabSnapshot>, Error> {
- let json: Value = postgrest
- .from(AF_COLLAB_SNAPSHOT_TABLE)
- .select(format!(
- "{},{},{},{}",
- AF_COLLAB_SNAPSHOT_ID_COLUMN,
- AF_COLLAB_SNAPSHOT_BLOB_COLUMN,
- AF_COLLAB_SNAPSHOT_CREATED_AT_COLUMN,
- AF_COLLAB_SNAPSHOT_ENCRYPT_COLUMN
- ))
- .order(format!("{}.desc", AF_COLLAB_SNAPSHOT_ID_COLUMN))
- .limit(limit)
- .eq(AF_COLLAB_SNAPSHOT_OID_COLUMN, object_id)
- .execute()
- .await?
- .get_json()
- .await?;
-
- let mut snapshots = vec![];
- let secret = postgrest.secret();
- match json.as_array() {
- None => {
- if let Some(snapshot) = parser_snapshot(object_id, &json, &secret) {
- snapshots.push(snapshot);
- }
- },
- Some(snapshot_values) => {
- for snapshot_value in snapshot_values {
- if let Some(snapshot) = parser_snapshot(object_id, snapshot_value, &secret) {
- snapshots.push(snapshot);
- }
- }
- },
- }
- Ok(snapshots)
-}
-
-fn parser_snapshot(
- object_id: &str,
- snapshot: &Value,
- secret: &Option<String>,
-) -> Option<RemoteCollabSnapshot> {
- let blob = match (
- snapshot
- .get(AF_COLLAB_SNAPSHOT_ENCRYPT_COLUMN)
- .and_then(|encrypt| encrypt.as_i64()),
- snapshot
- .get(AF_COLLAB_SNAPSHOT_BLOB_COLUMN)
- .and_then(|value| value.as_str()),
- ) {
- (Some(encrypt), Some(value)) => {
- SupabaseBinaryColumnDecoder::decode::<_, BinaryColumnDecoder>(value, encrypt as i32, secret)
- .ok()
- },
- _ => None,
- }?;
-
- let sid = snapshot.get("sid").and_then(|id| id.as_i64())?;
- let created_at = snapshot.get("created_at").and_then(|created_at| {
- created_at
- .as_str()
- .map(|id| DateTime::<Utc>::from_str(id).ok())
- .and_then(|date| date)
- })?;
-
- Some(RemoteCollabSnapshot {
- sid,
- oid: object_id.to_string(),
- blob,
- created_at: created_at.timestamp(),
- })
-}
-
-pub async fn batch_get_updates_from_server(
- object_ids: Vec<String>,
- object_ty: &CollabType,
- postgrest: Arc<PostgresWrapper>,
-) -> Result<CollabDocStateByOid, Error> {
- let json = postgrest
- .from(table_name(object_ty))
- .select("oid, key, value, encrypt, md5")
- .order(format!("{}.asc", AF_COLLAB_KEY_COLUMN))
- .in_("oid", object_ids)
- .execute()
- .await?
- .get_json()
- .await?;
-
- let mut updates_by_oid = CollabDocStateByOid::new();
- if let Some(records) = json.as_array() {
- for record in records {
- tracing::debug!("get updates from server: {:?}", record);
- if let Some(oid) = record.get("oid").and_then(|value| value.as_str()) {
- match parser_updates_form_json(record.clone(), &postgrest.secret()) {
- Ok(items) => {
- if items.is_empty() {
- updates_by_oid.insert(oid.to_string(), DataSource::Disk);
- } else {
- let updates = items.into_iter().map(|update| update.value);
-
- let doc_state = merge_updates_v1(updates)
- .map_err(|err| anyhow::anyhow!("merge updates failed: {:?}", err))?;
- updates_by_oid.insert(oid.to_string(), DataSource::DocStateV1(doc_state));
- }
- },
- Err(e) => {
- tracing::error!("parser_updates_form_json error: {:?}", e);
- },
- }
- }
- }
- }
- Ok(updates_by_oid)
-}
-
-pub async fn get_updates_from_server(
- object_id: &str,
- object_ty: &CollabType,
- postgrest: &Arc<PostgresWrapper>,
-) -> Result<Vec<UpdateItem>, Error> {
- let json = postgrest
- .from(table_name(object_ty))
- .select("key, value, encrypt, md5")
- .order(format!("{}.asc", AF_COLLAB_KEY_COLUMN))
- .eq("oid", object_id)
- .execute()
- .await?
- .get_json()
- .await?;
- parser_updates_form_json(json, &postgrest.secret())
-}
-
-/// json format:
-/// ```json
-/// [
-/// {
-/// "value": "\\x...",
-/// "encrypt": 1,
-/// "md5": "..."
-/// },
-/// {
-/// "value": "\\x...",
-/// "encrypt": 1,
-/// "md5": "..."
-/// },
-/// ...
-/// ]
-/// ```
-fn parser_updates_form_json(
- json: Value,
- encryption_secret: &Option<String>,
-) -> Result<Vec<UpdateItem>, Error> {
- let mut updates = vec![];
- match json.as_array() {
- None => {
- updates.push(parser_update_from_json(&json, encryption_secret)?);
- },
- Some(values) => {
- let expected_update_len = values.len();
- for value in values {
- updates.push(parser_update_from_json(value, encryption_secret)?);
- }
- if updates.len() != expected_update_len {
- return Err(anyhow::anyhow!(
- "The length of the updates does not match the length of the expected updates, indicating that some updates failed to parse."
- ));
- }
- },
- }
-
- Ok(updates)
-}
-
-/// Parses update from a JSON representation.
-///
-/// This function attempts to decode an encrypted value from a JSON object
-/// and verify its integrity against a provided MD5 hash.
-///
-/// # Parameters
-/// - `json`: The JSON value representing the update information.
-/// - `encryption_secret`: An optional encryption secret used for decrypting the value.
-///
-/// json format:
-/// ```json
-/// {
-/// "value": "\\x...",
-/// "encrypt": 1,
-/// "md5": "..."
-/// },
-/// ```
-fn parser_update_from_json(
- json: &Value,
- encryption_secret: &Option<String>,
-) -> Result<UpdateItem, Error> {
- let some_record = match (
- json.get("encrypt").and_then(|encrypt| encrypt.as_i64()),
- json.get("value").and_then(|value| value.as_str()),
- ) {
- (Some(encrypt), Some(value)) => {
- match SupabaseBinaryColumnDecoder::decode::<_, BinaryColumnDecoder>(
- value,
- encrypt as i32,
- encryption_secret,
- ) {
- Ok(value) => Some(value),
- Err(err) => {
- tracing::error!("Decode value column failed: {:?}", err);
- None
- },
- }
- },
- _ => None,
- };
-
- let some_key = json.get("key").and_then(|value| value.as_i64());
- if let (Some(value), Some(key)) = (some_record, some_key) {
- // Check the md5 of the value that we received from the server is equal to the md5 of the value
- // that we calculated locally.
- if let Some(expected_md5) = json.get("md5").and_then(|v| v.as_str()) {
- let value_md5 = md5(&value);
- if value_md5 != expected_md5 {
- let msg = format!(
- "md5 not match: key:{} {} != {}",
- key, value_md5, expected_md5
- );
- tracing::error!("{}", msg);
- return Err(anyhow::anyhow!(msg));
- }
- }
- Ok(UpdateItem { key, value })
- } else {
- let keys = json
- .as_object()
- .map(|map| map.iter().map(|(key, _)| key).collect::<Vec<&String>>());
- Err(anyhow::anyhow!(
- "missing key or value column. Current keys:: {:?}",
- keys
- ))
- }
-}
-
-pub struct UpdateItem {
- pub key: i64,
- pub value: Vec<u8>,
-}
-
-pub struct RetryCondition(pub Weak<PostgresWrapper>);
-impl Condition<anyhow::Error> for RetryCondition {
- fn should_retry(&mut self, _error: &anyhow::Error) -> bool {
- self.0.upgrade().is_some()
- }
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/api/user.rs b/frontend/rust-lib/flowy-server/src/supabase/api/user.rs
deleted file mode 100644
index 3712307af4..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/api/user.rs
+++ /dev/null
@@ -1,718 +0,0 @@
-use std::collections::HashMap;
-use std::future::Future;
-use std::iter::Take;
-use std::pin::Pin;
-use std::sync::{Arc, Weak};
-use std::time::Duration;
-
-use anyhow::Error;
-use arc_swap::ArcSwapOption;
-use collab::core::origin::CollabOrigin;
-use collab::preclude::Collab;
-use collab_entity::{CollabObject, CollabType};
-use serde_json::Value;
-use tokio::sync::oneshot::channel;
-use tokio_retry::strategy::FixedInterval;
-use tokio_retry::{Action, RetryIf};
-use uuid::Uuid;
-
-use flowy_error::{internal_error, FlowyError};
-use flowy_folder_pub::cloud::{Folder, FolderData, Workspace};
-use flowy_user_pub::cloud::*;
-use flowy_user_pub::entities::*;
-use flowy_user_pub::DEFAULT_USER_NAME;
-
-use lib_infra::box_any::BoxAny;
-use lib_infra::future::FutureResult;
-
-use crate::response::ExtendedResponse;
-use crate::supabase::api::request::{
- get_updates_from_server, FetchObjectUpdateAction, RetryCondition,
-};
-use crate::supabase::api::util::{
- InsertParamsBuilder, RealtimeBinaryColumnDecoder, SupabaseBinaryColumnDecoder,
-};
-use crate::supabase::api::{flush_collab_with_update, PostgresWrapper, SupabaseServerService};
-use crate::supabase::define::*;
-use crate::supabase::entities::UserProfileResponse;
-use crate::supabase::entities::{GetUserProfileParams, RealtimeUserEvent};
-use crate::supabase::entities::{RealtimeCollabUpdateEvent, RealtimeEvent, UidResponse};
-use crate::supabase::CollabUpdateSenderByOid;
-use crate::AppFlowyEncryption;
-
-pub struct SupabaseUserServiceImpl<T> {
- server: T,
- realtime_event_handlers: Vec<Box<dyn RealtimeEventHandler>>,
- user_update_rx: ArcSwapOption<UserUpdateReceiver>,
-}
-
-impl<T> SupabaseUserServiceImpl<T> {
- pub fn new(
- server: T,
- realtime_event_handlers: Vec<Box<dyn RealtimeEventHandler>>,
- user_update_rx: Option<UserUpdateReceiver>,
- ) -> Self {
- Self {
- server,
- realtime_event_handlers,
- user_update_rx: ArcSwapOption::from(user_update_rx.map(Arc::new)),
- }
- }
-}
-
-impl<T> UserCloudService for SupabaseUserServiceImpl<T>
-where
- T: SupabaseServerService,
-{
- fn sign_up(&self, params: BoxAny) -> FutureResult<AuthResponse, FlowyError> {
- let try_get_postgrest = self.server.try_get_postgrest();
- FutureResult::new(async move {
- let postgrest = try_get_postgrest?;
- let params = oauth_params_from_box_any(params)?;
- let is_new_user = postgrest
- .from(USER_TABLE)
- .select("uid")
- .eq("uuid", params.uuid.to_string())
- .execute()
- .await?
- .get_value::<Vec<UidResponse>>()
- .await?
- .is_empty();
-
- // Insert the user if it's a new user. After the user is inserted, we can query the user profile
- // and workspaces. The profile and workspaces are created by the database trigger.
- if is_new_user {
- let insert_params = InsertParamsBuilder::new()
- .insert(USER_UUID, params.uuid.to_string())
- .insert(USER_EMAIL, params.email)
- .build();
- let resp = postgrest
- .from(USER_TABLE)
- .insert(insert_params)
- .execute()
- .await?
- .success_with_body()
- .await?;
- tracing::debug!("Create user response: {:?}", resp);
- }
-
- // Query the user profile and workspaces
- tracing::debug!("user uuid: {}", params.uuid);
- let user_profile =
- get_user_profile(postgrest.clone(), GetUserProfileParams::Uuid(params.uuid))
- .await?
- .unwrap();
- let user_workspaces = get_user_workspaces(postgrest.clone(), user_profile.uid).await?;
- let latest_workspace = user_workspaces
- .iter()
- .find(|user_workspace| user_workspace.id == user_profile.latest_workspace_id)
- .cloned();
-
- let user_name = if user_profile.name.is_empty() {
- DEFAULT_USER_NAME()
- } else {
- user_profile.name
- };
-
- Ok(AuthResponse {
- user_id: user_profile.uid,
- user_uuid: params.uuid,
- name: user_name,
- latest_workspace: latest_workspace.unwrap(),
- user_workspaces,
- is_new_user,
- email: Some(user_profile.email),
- token: None,
- encryption_type: EncryptionType::from_sign(&user_profile.encryption_sign),
- updated_at: user_profile.updated_at.timestamp(),
- metadata: None,
- })
- })
- }
-
- fn sign_in(&self, params: BoxAny) -> FutureResult<AuthResponse, FlowyError> {
- let try_get_postgrest = self.server.try_get_postgrest();
- FutureResult::new(async move {
- let postgrest = try_get_postgrest?;
- let params = oauth_params_from_box_any(params)?;
- let uuid = params.uuid;
- let response = get_user_profile(postgrest.clone(), GetUserProfileParams::Uuid(uuid))
- .await?
- .unwrap();
- let user_workspaces = get_user_workspaces(postgrest.clone(), response.uid).await?;
- let latest_workspace = user_workspaces
- .iter()
- .find(|user_workspace| user_workspace.id == response.latest_workspace_id)
- .cloned();
-
- Ok(AuthResponse {
- user_id: response.uid,
- user_uuid: params.uuid,
- name: DEFAULT_USER_NAME(),
- latest_workspace: latest_workspace.unwrap(),
- user_workspaces,
- is_new_user: false,
- email: None,
- token: None,
- encryption_type: EncryptionType::from_sign(&response.encryption_sign),
- updated_at: response.updated_at.timestamp(),
- metadata: None,
- })
- })
- }
-
- fn sign_out(&self, _token: Option<String>) -> FutureResult<(), FlowyError> {
- FutureResult::new(async { Ok(()) })
- }
-
- fn generate_sign_in_url_with_email(&self, _email: &str) -> FutureResult<String, FlowyError> {
- FutureResult::new(async {
- Err(FlowyError::internal().with_context("Can't generate callback url when using supabase"))
- })
- }
-
- fn create_user(&self, _email: &str, _password: &str) -> FutureResult<(), FlowyError> {
- FutureResult::new(async {
- Err(FlowyError::not_support().with_context("Can't create user when using supabase"))
- })
- }
-
- fn sign_in_with_password(
- &self,
- _email: &str,
- _password: &str,
- ) -> FutureResult<UserProfile, FlowyError> {
- FutureResult::new(async {
- Err(FlowyError::not_support().with_context("Can't sign in with password when using supabase"))
- })
- }
-
- fn sign_in_with_magic_link(
- &self,
- _email: &str,
- _redirect_to: &str,
- ) -> FutureResult<(), FlowyError> {
- FutureResult::new(async {
- Err(
- FlowyError::not_support().with_context("Can't sign in with magic link when using supabase"),
- )
- })
- }
-
- fn sign_in_with_passcode(
- &self,
- _email: &str,
- _passcode: &str,
- ) -> FutureResult<GotrueTokenResponse, FlowyError> {
- FutureResult::new(async {
- Err(FlowyError::not_support().with_context("Can't sign in with passcode when using supabase"))
- })
- }
-
- fn generate_oauth_url_with_provider(&self, _provider: &str) -> FutureResult<String, FlowyError> {
- FutureResult::new(async {
- Err(FlowyError::internal().with_context("Can't generate oauth url when using supabase"))
- })
- }
-
- fn update_user(
- &self,
- _credential: UserCredentials,
- params: UpdateUserProfileParams,
- ) -> FutureResult<(), FlowyError> {
- let try_get_postgrest = self.server.try_get_postgrest();
- FutureResult::new(async move {
- let postgrest = try_get_postgrest?;
- update_user_profile(postgrest, params).await?;
- Ok(())
- })
- }
-
- fn get_user_profile(&self, credential: UserCredentials) -> FutureResult<UserProfile, FlowyError> {
- let try_get_postgrest = self.server.try_get_postgrest();
- let uid = credential
- .uid
- .ok_or(anyhow::anyhow!("uid is required"))
- .unwrap();
- FutureResult::new(async move {
- let postgrest = try_get_postgrest?;
- let user_profile_resp = get_user_profile(postgrest, GetUserProfileParams::Uid(uid)).await?;
- match user_profile_resp {
- None => Err(FlowyError::record_not_found()),
- Some(response) => Ok(UserProfile {
- uid: response.uid,
- email: response.email,
- name: response.name,
- token: "".to_string(),
- icon_url: "".to_string(),
- openai_key: "".to_string(),
- stability_ai_key: "".to_string(),
- workspace_id: response.latest_workspace_id,
- authenticator: Authenticator::Supabase,
- encryption_type: EncryptionType::from_sign(&response.encryption_sign),
- updated_at: response.updated_at.timestamp(),
- ai_model: "".to_string(),
- }),
- }
- })
- }
-
- fn open_workspace(&self, _workspace_id: &str) -> FutureResult<UserWorkspace, FlowyError> {
- FutureResult::new(async {
- Err(FlowyError::not_support().with_context("supabase server doesn't support open workspace"))
- })
- }
-
- fn get_all_workspace(&self, uid: i64) -> FutureResult<Vec<UserWorkspace>, FlowyError> {
- let try_get_postgrest = self.server.try_get_postgrest();
- FutureResult::new(async move {
- let postgrest = try_get_postgrest?;
- let user_workspaces = get_user_workspaces(postgrest, uid).await?;
- Ok(user_workspaces)
- })
- }
-
- fn get_user_awareness_doc_state(
- &self,
- _uid: i64,
- _workspace_id: &str,
- object_id: &str,
- ) -> FutureResult<Vec<u8>, FlowyError> {
- let try_get_postgrest = self.server.try_get_weak_postgrest();
- let (tx, rx) = channel();
- let object_id = object_id.to_string();
- tokio::spawn(async move {
- tx.send(
- async move {
- let postgrest = try_get_postgrest?;
- let action =
- FetchObjectUpdateAction::new(object_id, CollabType::UserAwareness, postgrest);
- action.run_with_fix_interval(3, 3).await
- }
- .await,
- )
- });
- FutureResult::new(async {
- let doc_state = rx.await.map_err(internal_error)?;
- doc_state.map_err(internal_error)
- })
- }
-
- fn receive_realtime_event(&self, json: Value) {
- match serde_json::from_value::<RealtimeEvent>(json) {
- Ok(event) => {
- tracing::trace!("Realtime event: {}", event);
- for handler in &self.realtime_event_handlers {
- if event.table.as_str().starts_with(handler.table_name()) {
- handler.handler_event(&event);
- }
- }
- },
- Err(e) => {
- tracing::error!("parser realtime event error: {}", e);
- },
- }
- }
-
- fn subscribe_user_update(&self) -> Option<UserUpdateReceiver> {
- let rx = self.user_update_rx.swap(None)?;
- Arc::into_inner(rx)
- }
-
- fn reset_workspace(&self, collab_object: CollabObject) -> FutureResult<(), FlowyError> {
- let try_get_postgrest = self.server.try_get_weak_postgrest();
- let (tx, rx) = channel();
- let init_update = default_workspace_doc_state(&collab_object);
- tokio::spawn(async move {
- tx.send(
- async move {
- let postgrest = try_get_postgrest?
- .upgrade()
- .ok_or(anyhow::anyhow!("postgrest is not available"))?;
-
- let updates = get_updates_from_server(
- &collab_object.object_id,
- &collab_object.collab_type,
- &postgrest,
- )
- .await?;
-
- flush_collab_with_update(
- &collab_object,
- updates,
- &postgrest,
- init_update,
- postgrest.secret(),
- )
- .await?;
- Ok(())
- }
- .await,
- )
- });
- FutureResult::new(async { rx.await? })
- }
-
- fn create_collab_object(
- &self,
- collab_object: &CollabObject,
- data: Vec<u8>,
- ) -> FutureResult<(), FlowyError> {
- let try_get_postgrest = self.server.try_get_weak_postgrest();
- let cloned_collab_object = collab_object.clone();
- let (tx, rx) = channel();
- tokio::spawn(async move {
- tx.send(
- async move {
- CreateCollabAction::new(cloned_collab_object, try_get_postgrest?, data)
- .run()
- .await?;
- Ok(())
- }
- .await,
- )
- });
- FutureResult::new(async { rx.await? })
- }
-
- fn batch_create_collab_object(
- &self,
- _workspace_id: &str,
- _objects: Vec<UserCollabParams>,
- ) -> FutureResult<(), FlowyError> {
- FutureResult::new(async {
- Err(
- FlowyError::local_version_not_support()
- .with_context("supabase server doesn't support batch create collab"),
- )
- })
- }
-
- fn create_workspace(&self, _workspace_name: &str) -> FutureResult<UserWorkspace, FlowyError> {
- FutureResult::new(async {
- Err(
- FlowyError::local_version_not_support()
- .with_context("supabase server doesn't support mulitple workspaces"),
- )
- })
- }
-
- fn delete_workspace(&self, _workspace_id: &str) -> FutureResult<(), FlowyError> {
- FutureResult::new(async {
- Err(
- FlowyError::local_version_not_support()
- .with_context("supabase server doesn't support mulitple workspaces"),
- )
- })
- }
-
- fn patch_workspace(
- &self,
- _workspace_id: &str,
- _new_workspace_name: Option<&str>,
- _new_workspace_icon: Option<&str>,
- ) -> FutureResult<(), FlowyError> {
- FutureResult::new(async {
- Err(
- FlowyError::local_version_not_support()
- .with_context("supabase server doesn't support mulitple workspaces"),
- )
- })
- }
-}
-
-pub struct CreateCollabAction {
- collab_object: CollabObject,
- postgrest: Weak<PostgresWrapper>,
- update: Vec<u8>,
-}
-
-impl CreateCollabAction {
- pub fn new(
- collab_object: CollabObject,
- postgrest: Weak<PostgresWrapper>,
- update: Vec<u8>,
- ) -> Self {
- Self {
- collab_object,
- postgrest,
- update,
- }
- }
-
- pub fn run(self) -> RetryIf<Take<FixedInterval>, CreateCollabAction, RetryCondition> {
- let postgrest = self.postgrest.clone();
- let retry_strategy = FixedInterval::new(Duration::from_secs(2)).take(3);
- RetryIf::spawn(retry_strategy, self, RetryCondition(postgrest))
- }
-}
-
-impl Action for CreateCollabAction {
- type Future = Pin<Box<dyn Future<Output = Result<Self::Item, Self::Error>> + Send>>;
- type Item = ();
- type Error = anyhow::Error;
-
- fn run(&mut self) -> Self::Future {
- let weak_postgres = self.postgrest.clone();
- let cloned_collab_object = self.collab_object.clone();
- let cloned_update = self.update.clone();
- Box::pin(async move {
- match weak_postgres.upgrade() {
- None => Ok(()),
- Some(postgrest) => {
- let secret = postgrest.secret();
- flush_collab_with_update(
- &cloned_collab_object,
- vec![],
- &postgrest,
- cloned_update,
- secret,
- )
- .await?;
- Ok(())
- },
- }
- })
- }
-}
-
-async fn get_user_profile(
- postgrest: Arc<PostgresWrapper>,
- params: GetUserProfileParams,
-) -> Result<Option<UserProfileResponse>, Error> {
- let mut builder = postgrest
- .from(USER_PROFILE_VIEW)
- .select("uid, email, name, encryption_sign, latest_workspace_id, updated_at");
-
- match params {
- GetUserProfileParams::Uid(uid) => builder = builder.eq("uid", uid.to_string()),
- GetUserProfileParams::Uuid(uuid) => builder = builder.eq("uuid", uuid.to_string()),
- }
-
- let mut profiles = builder
- .execute()
- .await?
- .error_for_status()?
- .get_value::<Vec<UserProfileResponse>>()
- .await?;
- match profiles.len() {
- 0 => Ok(None),
- 1 => Ok(Some(profiles.swap_remove(0))),
- _ => {
- tracing::error!("multiple user profile found");
- Ok(None)
- },
- }
-}
-
-async fn get_user_workspaces(
- postgrest: Arc<PostgresWrapper>,
- uid: i64,
-) -> Result<Vec<UserWorkspace>, Error> {
- postgrest
- .from(WORKSPACE_TABLE)
- .select("id:workspace_id, name:workspace_name, created_at, database_storage_id")
- .eq("owner_uid", uid.to_string())
- .execute()
- .await?
- .error_for_status()?
- .get_value::<Vec<UserWorkspace>>()
- .await
-}
-
-async fn update_user_profile(
- postgrest: Arc<PostgresWrapper>,
- params: UpdateUserProfileParams,
-) -> Result<(), Error> {
- if params.is_empty() {
- anyhow::bail!("no params to update");
- }
-
- // check if user exists
- let exists = !postgrest
- .from(USER_TABLE)
- .select("uid")
- .eq("uid", params.uid.to_string())
- .execute()
- .await?
- .error_for_status()?
- .get_value::<Vec<UidResponse>>()
- .await?
- .is_empty();
- if !exists {
- anyhow::bail!("user uid {} does not exist", params.uid);
- }
- let mut update_params = serde_json::Map::new();
- if let Some(name) = params.name {
- update_params.insert("name".to_string(), serde_json::json!(name));
- }
- if let Some(email) = params.email {
- update_params.insert("email".to_string(), serde_json::json!(email));
- }
- if let Some(encrypt_sign) = params.encryption_sign {
- update_params.insert(
- "encryption_sign".to_string(),
- serde_json::json!(encrypt_sign),
- );
- }
-
- let update_payload = serde_json::to_string(&update_params).unwrap();
- let resp = postgrest
- .from(USER_TABLE)
- .update(update_payload)
- .eq("uid", params.uid.to_string())
- .execute()
- .await?
- .success_with_body()
- .await?;
-
- tracing::trace!("update user profile resp: {:?}", resp);
- Ok(())
-}
-
-#[allow(dead_code)]
-async fn check_user(
- postgrest: Arc<PostgresWrapper>,
- uid: Option<i64>,
- uuid: Option<Uuid>,
-) -> Result<(), Error> {
- let mut builder = postgrest.from(USER_TABLE);
-
- if let Some(uid) = uid {
- builder = builder.eq("uid", uid.to_string());
- } else if let Some(uuid) = uuid {
- builder = builder.eq("uuid", uuid.to_string());
- } else {
- anyhow::bail!("uid or uuid is required");
- }
-
- let exists = !builder
- .execute()
- .await?
- .error_for_status()?
- .get_value::<Vec<UidResponse>>()
- .await?
- .is_empty();
- if !exists {
- anyhow::bail!("user does not exist, uid: {:?}, uuid: {:?}", uid, uuid);
- }
- Ok(())
-}
-
-pub trait RealtimeEventHandler: Send + Sync + 'static {
- fn table_name(&self) -> &str;
-
- fn handler_event(&self, event: &RealtimeEvent);
-}
-
-pub struct RealtimeUserHandler(pub UserUpdateSender);
-impl RealtimeEventHandler for RealtimeUserHandler {
- fn table_name(&self) -> &str {
- "af_user"
- }
-
- fn handler_event(&self, event: &RealtimeEvent) {
- if let Ok(user_event) = serde_json::from_value::<RealtimeUserEvent>(event.new.clone()) {
- let sender = self.0.clone();
- tokio::spawn(async move {
- let _ = sender
- .send(UserUpdate {
- uid: user_event.uid,
- name: Some(user_event.name),
- email: Some(user_event.email),
- encryption_sign: user_event.encryption_sign,
- })
- .await;
- });
- }
- }
-}
-
-pub struct RealtimeCollabUpdateHandler {
- sender_by_oid: Weak<CollabUpdateSenderByOid>,
- device_id: String,
- encryption: Weak<dyn AppFlowyEncryption>,
-}
-
-impl RealtimeCollabUpdateHandler {
- pub fn new(
- sender_by_oid: Weak<CollabUpdateSenderByOid>,
- device_id: String,
- encryption: Weak<dyn AppFlowyEncryption>,
- ) -> Self {
- Self {
- sender_by_oid,
- device_id,
- encryption,
- }
- }
-}
-impl RealtimeEventHandler for RealtimeCollabUpdateHandler {
- fn table_name(&self) -> &str {
- "af_collab_update"
- }
-
- fn handler_event(&self, event: &RealtimeEvent) {
- if let Ok(collab_update) =
- serde_json::from_value::<RealtimeCollabUpdateEvent>(event.new.clone())
- {
- if let Some(sender_by_oid) = self.sender_by_oid.upgrade() {
- if let Some(sender) = sender_by_oid.get(collab_update.oid.as_str()) {
- tracing::trace!(
- "current device: {}, event device: {}",
- self.device_id,
- collab_update.did.as_str()
- );
- if self.device_id != collab_update.did {
- let encryption_secret = self
- .encryption
- .upgrade()
- .and_then(|encryption| encryption.get_secret());
-
- tracing::trace!(
- "Parse collab update with len: {}, encrypt: {}",
- collab_update.value.len(),
- collab_update.encrypt,
- );
-
- match SupabaseBinaryColumnDecoder::decode::<_, RealtimeBinaryColumnDecoder>(
- collab_update.value.as_str(),
- collab_update.encrypt,
- &encryption_secret,
- ) {
- Ok(value) => {
- if let Err(e) = sender.send(value) {
- tracing::debug!("send realtime update error: {}", e);
- }
- },
- Err(err) => {
- tracing::error!("decode collab update error: {}", err);
- },
- }
- }
- }
- }
- }
- }
-}
-
-fn default_workspace_doc_state(collab_object: &CollabObject) -> Vec<u8> {
- let workspace_id = collab_object.object_id.clone();
- let collab =
- Collab::new_with_origin(CollabOrigin::Empty, &collab_object.object_id, vec![], false);
- let workspace = Workspace::new(workspace_id, "My workspace".to_string(), collab_object.uid);
- let folder = Folder::open_with(
- collab_object.uid,
- collab,
- None,
- Some(FolderData::new(workspace)),
- );
- folder.encode_collab().unwrap().doc_state.to_vec()
-}
-
-fn oauth_params_from_box_any(any: BoxAny) -> Result<SupabaseOAuthParams, Error> {
- let map: HashMap<String, String> = any.unbox_or_error()?;
- let uuid = uuid_from_map(&map)?;
- let email = map.get("email").cloned().unwrap_or_default();
- Ok(SupabaseOAuthParams { uuid, email })
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/api/util.rs b/frontend/rust-lib/flowy-server/src/supabase/api/util.rs
deleted file mode 100644
index 1d02bfc87c..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/api/util.rs
+++ /dev/null
@@ -1,120 +0,0 @@
-use anyhow::Error;
-use anyhow::Result;
-use serde_json::Value;
-
-use flowy_encrypt::{decrypt_data, encrypt_data};
-
-#[derive(Default)]
-pub struct InsertParamsBuilder {
- map: serde_json::Map<String, Value>,
-}
-
-impl InsertParamsBuilder {
- pub fn new() -> Self {
- Self::default()
- }
-
- pub fn insert<T: serde::Serialize>(mut self, key: &str, value: T) -> Self {
- self
- .map
- .insert(key.to_string(), serde_json::to_value(value).unwrap());
- self
- }
-
- pub fn build(self) -> String {
- serde_json::to_string(&self.map).unwrap()
- }
-}
-
-/// An encoder for binary columns in Supabase.
-///
-/// Provides utilities to encode binary data into a format suitable for Supabase columns.
-pub struct SupabaseBinaryColumnEncoder;
-
-impl SupabaseBinaryColumnEncoder {
- /// Encodes the given binary data into a Supabase-friendly string representation.
- ///
- /// # Parameters
- /// - `value`: The binary data to encode.
- ///
- /// # Returns
- /// Returns the encoded string in the format: `\\xHEX_ENCODED_STRING`
- pub fn encode<T: AsRef<[u8]>>(
- value: T,
- encryption_secret: &Option<String>,
- ) -> Result<(String, i32)> {
- let encrypt = if encryption_secret.is_some() { 1 } else { 0 };
- let value = match encryption_secret {
- None => hex::encode(value),
- Some(encryption_secret) => {
- let encrypt_data = encrypt_data(value, encryption_secret)?;
- hex::encode(encrypt_data)
- },
- };
-
- Ok((format!("\\x{}", value), encrypt))
- }
-}
-
-/// A decoder for binary columns in Supabase.
-///
-/// Provides utilities to decode a string from Supabase columns back into binary data.
-pub struct SupabaseBinaryColumnDecoder;
-
-impl SupabaseBinaryColumnDecoder {
- /// Decodes a Supabase binary column string into binary data.
- ///
- /// # Parameters
- /// - `value`: The string representation from a Supabase binary column.
- ///
- /// # Returns
- /// Returns an `Option` containing the decoded binary data if decoding is successful.
- /// Otherwise, returns `None`.
- pub fn decode<T: AsRef<str>, D: HexDecoder>(
- value: T,
- encrypt: i32,
- encryption_secret: &Option<String>,
- ) -> Result<Vec<u8>> {
- let s = value
- .as_ref()
- .strip_prefix("\\x")
- .ok_or(anyhow::anyhow!("Value is not start with: \\x",))?;
-
- if encrypt == 0 {
- let bytes = D::decode(s)?;
- Ok(bytes)
- } else {
- match encryption_secret {
- None => Err(anyhow::anyhow!(
- "encryption_secret is None, but encrypt is 1"
- )),
- Some(encryption_secret) => {
- let encrypt_data = D::decode(s)?;
- decrypt_data(encrypt_data, encryption_secret)
- },
- }
- }
- }
-}
-
-pub trait HexDecoder {
- fn decode<T: AsRef<[u8]>>(data: T) -> Result<Vec<u8>, Error>;
-}
-
-pub struct RealtimeBinaryColumnDecoder;
-impl HexDecoder for RealtimeBinaryColumnDecoder {
- fn decode<T: AsRef<[u8]>>(data: T) -> Result<Vec<u8>, Error> {
- // The realtime event binary column string is encoded twice. So it needs to be decoded twice.
- let bytes = hex::decode(data)?;
- let bytes = hex::decode(bytes)?;
- Ok(bytes)
- }
-}
-
-pub struct BinaryColumnDecoder;
-impl HexDecoder for BinaryColumnDecoder {
- fn decode<T: AsRef<[u8]>>(data: T) -> Result<Vec<u8>, Error> {
- let bytes = hex::decode(data)?;
- Ok(bytes)
- }
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/define.rs b/frontend/rust-lib/flowy-server/src/supabase/define.rs
deleted file mode 100644
index 1067c999d8..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/define.rs
+++ /dev/null
@@ -1,49 +0,0 @@
-use collab_entity::CollabType;
-
-pub const AF_COLLAB_UPDATE_TABLE: &str = "af_collab_update";
-pub const AF_COLLAB_KEY_COLUMN: &str = "key";
-pub const AF_COLLAB_SNAPSHOT_OID_COLUMN: &str = "oid";
-pub const AF_COLLAB_SNAPSHOT_ID_COLUMN: &str = "sid";
-pub const AF_COLLAB_SNAPSHOT_BLOB_COLUMN: &str = "blob";
-pub const AF_COLLAB_SNAPSHOT_ENCRYPT_COLUMN: &str = "encrypt";
-pub const AF_COLLAB_SNAPSHOT_BLOB_SIZE_COLUMN: &str = "blob_size";
-pub const AF_COLLAB_SNAPSHOT_CREATED_AT_COLUMN: &str = "created_at";
-pub const AF_COLLAB_SNAPSHOT_TABLE: &str = "af_collab_snapshot";
-
-pub const USER_UUID: &str = "uuid";
-pub const USER_SIGN_IN_URL: &str = "sign_in_url";
-pub const USER_EMAIL: &str = "email";
-pub const USER_DEVICE_ID: &str = "device_id";
-
-pub const USER_TABLE: &str = "af_user";
-pub const USER_UID: &str = "uid";
-pub const OWNER_USER_UID: &str = "owner_uid";
-pub const WORKSPACE_TABLE: &str = "af_workspace";
-pub const USER_PROFILE_VIEW: &str = "af_user_profile_view";
-
-pub(crate) const WORKSPACE_ID: &str = "workspace_id";
-pub(crate) const WORKSPACE_NAME: &str = "workspace_name";
-pub(crate) const CREATED_AT: &str = "created_at";
-
-pub fn table_name(ty: &CollabType) -> String {
- match ty {
- CollabType::DatabaseRow => format!("{}_database_row", AF_COLLAB_UPDATE_TABLE),
- CollabType::Document | CollabType::Unknown => format!("{}_document", AF_COLLAB_UPDATE_TABLE),
- CollabType::Database => format!("{}_database", AF_COLLAB_UPDATE_TABLE),
- CollabType::WorkspaceDatabase => format!("{}_w_database", AF_COLLAB_UPDATE_TABLE),
- CollabType::Folder => format!("{}_folder", AF_COLLAB_UPDATE_TABLE),
- CollabType::UserAwareness => format!("{}_user_awareness", AF_COLLAB_UPDATE_TABLE),
- }
-}
-
-pub fn partition_key(collab_type: &CollabType) -> i32 {
- match collab_type {
- CollabType::Document => 0,
- CollabType::Database => 1,
- CollabType::WorkspaceDatabase => 2,
- CollabType::Folder => 3,
- CollabType::DatabaseRow => 4,
- CollabType::UserAwareness => 5,
- CollabType::Unknown => 0,
- }
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/entities.rs b/frontend/rust-lib/flowy-server/src/supabase/entities.rs
deleted file mode 100644
index 80e5f87511..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/entities.rs
+++ /dev/null
@@ -1,79 +0,0 @@
-use std::fmt;
-use std::fmt::Display;
-
-use chrono::{DateTime, Utc};
-use serde::Deserialize;
-use serde_json::Value;
-use uuid::Uuid;
-
-use crate::util::deserialize_null_or_default;
-
-pub enum GetUserProfileParams {
- Uid(i64),
- Uuid(Uuid),
-}
-
-#[allow(dead_code)]
-#[derive(Debug, Deserialize, Clone)]
-pub(crate) struct UserProfileResponse {
- pub uid: i64,
- #[serde(deserialize_with = "deserialize_null_or_default")]
- pub name: String,
-
- #[serde(deserialize_with = "deserialize_null_or_default")]
- pub email: String,
-
- #[serde(deserialize_with = "deserialize_null_or_default")]
- pub latest_workspace_id: String,
-
- #[serde(deserialize_with = "deserialize_null_or_default")]
- pub encryption_sign: String,
-
- pub updated_at: DateTime<Utc>,
-}
-
-#[derive(Deserialize, Clone)]
-pub(crate) struct UidResponse {
- #[allow(dead_code)]
- pub uid: i64,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct RealtimeEvent {
- pub schema: String,
- pub table: String,
- #[serde(rename = "eventType")]
- pub event_type: String,
- pub new: Value,
-}
-impl Display for RealtimeEvent {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- write!(
- f,
- "schema: {}, table: {}, event_type: {}",
- self.schema, self.table, self.event_type
- )
- }
-}
-
-#[derive(Debug, Deserialize)]
-pub struct RealtimeCollabUpdateEvent {
- pub oid: String,
- pub uid: i64,
- pub key: i64,
- pub did: String,
- pub value: String,
- #[serde(default)]
- pub encrypt: i32,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct RealtimeUserEvent {
- pub uid: i64,
- #[serde(deserialize_with = "deserialize_null_or_default")]
- pub name: String,
- #[serde(deserialize_with = "deserialize_null_or_default")]
- pub email: String,
- #[serde(deserialize_with = "deserialize_null_or_default")]
- pub encryption_sign: String,
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/file_storage/builder.rs b/frontend/rust-lib/flowy-server/src/supabase/file_storage/builder.rs
deleted file mode 100644
index 6db01f1cc6..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/file_storage/builder.rs
+++ /dev/null
@@ -1,154 +0,0 @@
-use anyhow::Error;
-use flowy_storage_pub::cloud::StorageObject;
-use hyper::header::CONTENT_TYPE;
-use reqwest::header::IntoHeaderName;
-use reqwest::multipart::{Form, Part};
-use reqwest::{
- header::{HeaderMap, HeaderValue},
- Client, Method, RequestBuilder,
-};
-use std::borrow::Cow;
-use tokio::fs::File;
-use tokio::io::AsyncReadExt;
-use url::Url;
-
-use crate::supabase::file_storage::{DeleteObjects, FileOptions, NewBucket, RequestBody};
-
-#[allow(dead_code)]
-pub struct StorageRequestBuilder {
- pub url: Url,
- headers: HeaderMap,
- client: Client,
- method: Method,
- body: RequestBody,
-}
-
-#[allow(dead_code)]
-impl StorageRequestBuilder {
- pub fn new(url: Url, headers: HeaderMap, client: Client) -> Self {
- Self {
- url,
- headers,
- client,
- method: Method::GET,
- body: RequestBody::Empty,
- }
- }
- pub fn with_header(mut self, key: impl IntoHeaderName, value: HeaderValue) -> Self {
- self.headers.insert(key, value);
- self
- }
-
- pub fn get_buckets(mut self) -> Self {
- self.method = Method::GET;
- self.url.path_segments_mut().unwrap().push("bucket");
- self
- }
-
- pub fn create_bucket(mut self, bucket_name: &str) -> Self {
- self.method = Method::POST;
- self
- .headers
- .insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
- self.url.path_segments_mut().unwrap().push("bucket");
- let bucket = serde_json::to_string(&NewBucket::new(bucket_name.to_string())).unwrap();
- self.body = RequestBody::BodyString { text: bucket };
- self
- }
-
- pub fn delete_object(mut self, bucket_id: &str, object: &str) -> Self {
- self.method = Method::DELETE;
- self
- .headers
- .insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
- let delete_objects = DeleteObjects::new(vec![object.to_string()]);
- let text = serde_json::to_string(&delete_objects).unwrap();
- self.body = RequestBody::BodyString { text };
- self
- .url
- .path_segments_mut()
- .unwrap()
- .push("object")
- .push(bucket_id)
- .push(object);
- self
- }
-
- pub fn get_object(mut self, bucket_name: &str, object: &str) -> Self {
- self.method = Method::GET;
- self
- .url
- .path_segments_mut()
- .unwrap()
- .push("object")
- .push(bucket_name)
- .push(object);
- self
- }
-
- pub fn upload_object(mut self, bucket_name: &str, object: StorageObject) -> Self {
- self.method = Method::POST;
- let options = FileOptions::from_mime(object.value.mime_type());
- self
- .url
- .path_segments_mut()
- .unwrap()
- .push("object")
- .push(bucket_name)
- .push(&object.file_name);
-
- self.body = (options, object.value).into();
-
- self
- }
-
- pub async fn build(mut self) -> Result<RequestBuilder, Error> {
- let url = self.url.to_string();
- let mut builder = self.client.request(self.method, url);
- match self.body {
- RequestBody::Empty => {
- // Do nothing
- },
- RequestBody::MultiPartFile { file_path, options } => {
- self.headers.insert(
- "x-upsert",
- HeaderValue::from_str(&options.upsert.to_string()).unwrap(),
- );
-
- let mut file = File::open(&file_path).await?;
- let mut buffer = Vec::new();
- file.read_to_end(&mut buffer).await?;
-
- let part = Part::bytes(buffer)
- .file_name(file_path.to_string())
- .mime_str(&options.content_type)?;
-
- let form = Form::new()
- .part(file_path, part)
- .text("cacheControl", options.cache_control);
-
- builder = builder.multipart(form);
- },
- RequestBody::MultiPartBytes { bytes, options } => {
- self.headers.insert(
- "x-upsert",
- HeaderValue::from_str(&options.upsert.to_string()).unwrap(),
- );
- let part = Part::bytes(Cow::Owned(bytes.to_vec()))
- .file_name("")
- .mime_str(&options.content_type)?;
-
- let form = Form::new()
- .part("", part)
- .text("cacheControl", options.cache_control);
-
- builder = builder.multipart(form);
- },
- RequestBody::BodyString { text } => {
- builder = builder.body(text);
- },
- }
- builder = builder.headers(self.headers);
- Ok(builder)
- }
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/file_storage/core.rs b/frontend/rust-lib/flowy-server/src/supabase/file_storage/core.rs
deleted file mode 100644
index f150084c2d..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/file_storage/core.rs
+++ /dev/null
@@ -1,76 +0,0 @@
-#![allow(clippy::all)]
-#![allow(unknown_lints)]
-#![allow(unused_attributes)]
-use std::sync::Weak;
-
-use anyhow::{anyhow, Error};
-use url::Url;
-
-use crate::AppFlowyEncryption;
-use flowy_encrypt::{decrypt_data, encrypt_data};
-
-#[allow(dead_code)]
-struct ObjectEncryption {
- encryption: Weak<dyn AppFlowyEncryption>,
-}
-
-impl ObjectEncryption {
- #[allow(dead_code)]
- fn new(encryption: Weak<dyn AppFlowyEncryption>) -> Self {
- Self { encryption }
- }
-
- #[allow(dead_code)]
- fn encrypt(&self, object_data: Vec<u8>) -> Result<Vec<u8>, Error> {
- if let Some(secret) = self
- .encryption
- .upgrade()
- .and_then(|encryption| encryption.get_secret())
- {
- let encryption_data = encrypt_data(object_data, &secret)?;
- Ok(encryption_data)
- } else {
- Ok(object_data)
- }
- }
-
- #[allow(dead_code)]
- fn decrypt(&self, object_data: Vec<u8>) -> Result<Vec<u8>, Error> {
- if let Some(secret) = self
- .encryption
- .upgrade()
- .and_then(|encryption| encryption.get_secret())
- {
- let decryption_data = decrypt_data(object_data, &secret)?;
- Ok(decryption_data)
- } else {
- Ok(object_data)
- }
- }
-}
-
-#[allow(dead_code)]
-struct ObjectLocation<'a> {
- bucket_id: &'a str,
- file_name: &'a str,
-}
-
-#[allow(dead_code)]
-fn get_object_location_from(url: &Url) -> Result<ObjectLocation, Error> {
- let mut segments = url
- .path_segments()
- .ok_or(anyhow!("Invalid object url: {}", url))?
- .collect::<Vec<_>>();
-
- let file_name = segments
- .pop()
- .ok_or(anyhow!("Can't get file name from url: {}", url))?;
- let bucket_id = segments
- .pop()
- .ok_or(anyhow!("Can't get bucket id from url: {}", url))?;
-
- Ok(ObjectLocation {
- bucket_id,
- file_name,
- })
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/file_storage/entities.rs b/frontend/rust-lib/flowy-server/src/supabase/file_storage/entities.rs
deleted file mode 100644
index ec1ffa6c7b..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/file_storage/entities.rs
+++ /dev/null
@@ -1,107 +0,0 @@
-use bytes::Bytes;
-use flowy_storage_pub::cloud::ObjectValueSupabase;
-use serde::{Deserialize, Serialize};
-
-use crate::supabase;
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct SupabaseStorageError {
- pub status_code: String,
- pub error: String,
- pub message: String,
-}
-
-#[derive(Serialize)]
-pub struct NewBucket {
- pub name: String,
- pub id: String,
- pub public: bool,
- pub file_size_limit: Option<u32>,
- pub allowed_mime_types: Option<Vec<String>>,
-}
-
-impl NewBucket {
- pub fn new(name: String) -> Self {
- Self {
- name: name.clone(),
- id: name,
- public: false,
- file_size_limit: None,
- allowed_mime_types: None,
- }
- }
-}
-
-pub struct FileOptions {
- pub cache_control: String,
- pub upsert: bool,
- pub content_type: String,
-}
-
-impl FileOptions {
- pub fn from_mime(mime: String) -> Self {
- Self {
- cache_control: "3600".to_string(),
- upsert: false,
- content_type: mime,
- }
- }
-
- pub fn with_cache_control(mut self, cache_control: &str) -> Self {
- self.cache_control = cache_control.to_string();
- self
- }
-
- pub fn with_upsert(mut self, upsert: bool) -> Self {
- self.upsert = upsert;
- self
- }
-
- pub fn with_content_type(mut self, content_type: &str) -> Self {
- self.content_type = content_type.to_string();
- self
- }
-}
-#[derive(Serialize)]
-pub struct DeleteObjects {
- pub prefixes: Vec<String>,
-}
-
-impl DeleteObjects {
- pub fn new(prefixes: Vec<String>) -> Self {
- Self { prefixes }
- }
-}
-
-pub enum RequestBody {
- Empty,
- MultiPartFile {
- file_path: String,
- options: FileOptions,
- },
- MultiPartBytes {
- bytes: Bytes,
- options: FileOptions,
- },
- BodyString {
- text: String,
- },
-}
-
-impl From<(FileOptions, ObjectValueSupabase)> for RequestBody {
- fn from(
- params: (
- supabase::file_storage::entities::FileOptions,
- ObjectValueSupabase,
- ),
- ) -> Self {
- let (options, value) = params;
- match value {
- ObjectValueSupabase::File { file_path } => RequestBody::MultiPartFile { file_path, options },
- ObjectValueSupabase::Bytes { bytes, mime: _ } => {
- RequestBody::MultiPartBytes { bytes, options }
- },
- }
- }
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/file_storage/mod.rs b/frontend/rust-lib/flowy-server/src/supabase/file_storage/mod.rs
deleted file mode 100644
index 5da091c22c..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/file_storage/mod.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-pub use entities::*;
-
-mod builder;
-pub mod core;
-mod entities;
diff --git a/frontend/rust-lib/flowy-server/src/supabase/file_storage/plan.rs b/frontend/rust-lib/flowy-server/src/supabase/file_storage/plan.rs
deleted file mode 100644
index 39a33c8853..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/file_storage/plan.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-use std::sync::Weak;
-
-use flowy_error::FlowyError;
-use flowy_storage_pub::cloud::{FileStoragePlan, StorageObject};
-use lib_infra::future::FutureResult;
-
-use crate::supabase::api::RESTfulPostgresServer;
-
-#[derive(Default)]
-pub struct FileStoragePlanImpl {
- #[allow(dead_code)]
- uid: Weak<Option<i64>>,
- #[allow(dead_code)]
- postgrest: Option<Weak<RESTfulPostgresServer>>,
-}
-
-impl FileStoragePlanImpl {
- pub fn new(uid: Weak<Option<i64>>, postgrest: Option<Weak<RESTfulPostgresServer>>) -> Self {
- Self { uid, postgrest }
- }
-}
-
-impl FileStoragePlan for FileStoragePlanImpl {
- fn storage_size(&self) -> FutureResult<u64, FlowyError> {
- // 1 GB
- FutureResult::new(async { Ok(1024 * 1024 * 1024) })
- }
-
- fn maximum_file_size(&self) -> FutureResult<u64, FlowyError> {
- // 5 MB
- FutureResult::new(async { Ok(5 * 1024 * 1024) })
- }
-
- fn check_upload_object(&self, _object: &StorageObject) -> FutureResult<(), FlowyError> {
- FutureResult::new(async { Ok(()) })
- }
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/mod.rs b/frontend/rust-lib/flowy-server/src/supabase/mod.rs
deleted file mode 100644
index db0c6ae785..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/mod.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-pub use server::*;
-
-mod entities;
-// pub mod storage_impls;
-pub mod define;
-// mod queue;
-pub mod api;
-pub mod file_storage;
-mod server;
diff --git a/frontend/rust-lib/flowy-server/src/supabase/queue.rs b/frontend/rust-lib/flowy-server/src/supabase/queue.rs
deleted file mode 100644
index 15785f335e..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/queue.rs
+++ /dev/null
@@ -1,204 +0,0 @@
-use std::cmp::Ordering;
-use std::collections::BinaryHeap;
-use std::fmt::Debug;
-use std::marker::PhantomData;
-use std::ops::{Deref, DerefMut};
-use std::sync::{Arc, Weak};
-
-use tokio::sync::watch;
-
-use lib_infra::async_trait::async_trait;
-
-pub trait RequestPayload: Clone + Ord {}
-
-#[derive(Debug, Eq, PartialEq, Clone)]
-pub enum RequestState {
- Pending,
- Processing,
- Done,
-}
-
-#[derive(Debug)]
-pub struct PendingRequest<Payload> {
- pub payload: Payload,
- pub state: RequestState,
-}
-
-impl<Payload> PendingRequest<Payload> {
- pub fn new(payload: Payload) -> Self {
- Self {
- payload,
- state: RequestState::Pending,
- }
- }
-
- #[allow(dead_code)]
- pub fn state(&self) -> &RequestState {
- &self.state
- }
-
- pub fn set_state(&mut self, new_state: RequestState)
- where
- Payload: Debug,
- {
- if self.state != new_state {
- // tracing::trace!(
- // "PgRequest {:?} from {:?} to {:?}",
- // self.payload,
- // self.state,
- // new_state,
- // );
- self.state = new_state;
- }
- }
-
- pub fn is_processing(&self) -> bool {
- self.state == RequestState::Processing
- }
-
- pub fn is_done(&self) -> bool {
- self.state == RequestState::Done
- }
-}
-
-impl<Payload> Clone for PendingRequest<Payload>
-where
- Payload: Clone + Debug,
-{
- fn clone(&self) -> Self {
- Self {
- payload: self.payload.clone(),
- state: self.state.clone(),
- }
- }
-}
-
-pub(crate) struct RequestQueue<Payload>(BinaryHeap<PendingRequest<Payload>>);
-
-impl<Payload> RequestQueue<Payload>
-where
- Payload: Ord,
-{
- pub(crate) fn new() -> Self {
- Self(BinaryHeap::new())
- }
-}
-
-impl<Payload> Deref for RequestQueue<Payload>
-where
- Payload: Ord,
-{
- type Target = BinaryHeap<PendingRequest<Payload>>;
-
- fn deref(&self) -> &Self::Target {
- &self.0
- }
-}
-
-impl<Payload> DerefMut for RequestQueue<Payload>
-where
- Payload: Ord,
-{
- fn deref_mut(&mut self) -> &mut Self::Target {
- &mut self.0
- }
-}
-
-impl<Payload> Eq for PendingRequest<Payload> where Payload: Eq {}
-
-impl<Payload> PartialEq for PendingRequest<Payload>
-where
- Payload: PartialEq,
-{
- fn eq(&self, other: &Self) -> bool {
- self.payload == other.payload
- }
-}
-
-impl<Payload> PartialOrd for PendingRequest<Payload>
-where
- Payload: PartialOrd + Ord,
-{
- fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
- Some(self.cmp(other))
- }
-}
-
-impl<Payload> Ord for PendingRequest<Payload>
-where
- Payload: Ord,
-{
- fn cmp(&self, other: &Self) -> Ordering {
- self.payload.cmp(&other.payload)
- }
-}
-
-#[async_trait]
-pub trait RequestHandler<Payload>: Send + Sync + 'static {
- async fn prepare_request(&self) -> Option<PendingRequest<Payload>>;
- async fn handle_request(&self, request: PendingRequest<Payload>) -> Option<()>;
- fn notify(&self);
-}
-
-#[async_trait]
-impl<T, Payload> RequestHandler<Payload> for Arc<T>
-where
- T: RequestHandler<Payload>,
- Payload: 'static + Send + Sync,
-{
- async fn prepare_request(&self) -> Option<PendingRequest<Payload>> {
- (**self).prepare_request().await
- }
-
- async fn handle_request(&self, request: PendingRequest<Payload>) -> Option<()> {
- (**self).handle_request(request).await
- }
-
- fn notify(&self) {
- (**self).notify()
- }
-}
-
-pub struct RequestRunner<Payload>(PhantomData<Payload>);
-impl<Payload> RequestRunner<Payload>
-where
- Payload: 'static + Send + Sync,
-{
- pub async fn run(
- mut notifier: watch::Receiver<bool>,
- handler: Weak<dyn RequestHandler<Payload>>,
- ) {
- if let Some(handler) = handler.upgrade() {
- handler.notify();
- }
- loop {
- // stops the runner if the notifier was closed.
- if notifier.changed().await.is_err() {
- break;
- }
-
- // stops the runner if the value of notifier is `true`
- if *notifier.borrow() {
- break;
- }
-
- if let Some(handler) = handler.upgrade() {
- if let Some(request) = handler.prepare_request().await {
- if request.is_done() {
- handler.notify();
- continue;
- }
-
- if request.is_processing() {
- continue;
- }
-
- let _ = handler.handle_request(request).await;
- handler.notify();
- }
- } else {
- break;
- }
- }
- }
-}
diff --git a/frontend/rust-lib/flowy-server/src/supabase/server.rs b/frontend/rust-lib/flowy-server/src/supabase/server.rs
deleted file mode 100644
index 00dd46e8ba..0000000000
--- a/frontend/rust-lib/flowy-server/src/supabase/server.rs
+++ /dev/null
@@ -1,183 +0,0 @@
-use arc_swap::ArcSwapOption;
-use flowy_search_pub::cloud::SearchCloudService;
-use std::sync::{Arc, Weak};
-
-use collab_entity::CollabObject;
-use collab_plugins::cloud_storage::{RemoteCollabStorage, RemoteUpdateSender};
-use dashmap::DashMap;
-
-use flowy_database_pub::cloud::{DatabaseAIService, DatabaseCloudService};
-use flowy_document_pub::cloud::DocumentCloudService;
-use flowy_folder_pub::cloud::FolderCloudService;
-use flowy_server_pub::supabase_config::SupabaseConfiguration;
-use flowy_storage_pub::cloud::StorageCloudService;
-use flowy_user_pub::cloud::UserCloudService;
-
-use crate::supabase::api::{
- RESTfulPostgresServer, RealtimeCollabUpdateHandler, RealtimeEventHandler, RealtimeUserHandler,
- SupabaseCollabStorageImpl, SupabaseDatabaseServiceImpl, SupabaseDocumentServiceImpl,
- SupabaseFolderServiceImpl, SupabaseServerServiceImpl, SupabaseUserServiceImpl,
-};
-
-use crate::{AppFlowyEncryption, AppFlowyServer};
-
-/// https://www.pgbouncer.org/features.html
-/// Only support session mode.
-///
-/// Session mode:
-/// When a new client connects, a connection is assigned to the client until it disconnects. Afterward,
-/// the connection is returned back to the pool. All PostgreSQL features can be used with this option.
-/// For the moment, the default pool size of pgbouncer in supabase is 15 in session mode. Which means
-/// that we can have 15 concurrent connections to the database.
-///
-/// Transaction mode:
-/// This is the suggested option for serverless functions. With this, the connection is only assigned
-/// to the client for the duration of a transaction. Once done, the connection is returned to the pool.
-/// Two consecutive transactions from the same client could be done over two, different connections.
-/// Some session-based PostgreSQL features such as prepared statements are not available with this option.
-/// A more comprehensive list of incompatible features can be found here.
-///
-/// Most of the case, Session mode is faster than Transaction mode(no statement cache(https://github.com/supabase/supavisor/issues/69) and queue transaction).
-/// But Transaction mode is more suitable for serverless functions. It can reduce the number of concurrent
-/// connections to the database.
-/// TODO(nathan): fix prepared statement error when using transaction mode. https://github.com/prisma/prisma/issues/11643
-///
-#[derive(Clone, Debug, Default)]
-pub enum PgPoolMode {
- #[default]
- Session,
- Transaction,
-}
-
-impl PgPoolMode {
- pub fn support_prepare_cached(&self) -> bool {
- matches!(self, PgPoolMode::Session)
- }
-}
-
-pub type CollabUpdateSenderByOid = DashMap<String, RemoteUpdateSender>;
-/// Supabase server is used to provide the implementation of the [AppFlowyServer] trait.
-/// It contains the configuration of the supabase server and the postgres server.
-pub struct SupabaseServer {
- #[allow(dead_code)]
- config: SupabaseConfiguration,
- device_id: String,
- #[allow(dead_code)]
- uid: Arc<ArcSwapOption<i64>>,
- collab_update_sender: Arc<CollabUpdateSenderByOid>,
- restful_postgres: Arc<ArcSwapOption<RESTfulPostgresServer>>,
- encryption: Weak<dyn AppFlowyEncryption>,
-}
-
-impl SupabaseServer {
- pub fn new(
- uid: Arc<ArcSwapOption<i64>>,
- config: SupabaseConfiguration,
- enable_sync: bool,
- device_id: String,
- encryption: Weak<dyn AppFlowyEncryption>,
- ) -> Self {
- let collab_update_sender = Default::default();
- let restful_postgres = if enable_sync {
- Some(Arc::new(RESTfulPostgresServer::new(
- config.clone(),
- encryption.clone(),
- )))
- } else {
- None
- };
- Self {
- config,
- device_id,
- collab_update_sender,
- restful_postgres: Arc::new(ArcSwapOption::from(restful_postgres)),
- encryption,
- uid,
- }
- }
-}
-
-impl AppFlowyServer for SupabaseServer {
- fn set_enable_sync(&self, uid: i64, enable: bool) {
- tracing::info!("{} supabase sync: {}", uid, enable);
-
- if enable {
- self.restful_postgres.rcu(|old| match old {
- Some(existing) => Some(existing.clone()),
- None => {
- let postgres = Arc::new(RESTfulPostgresServer::new(
- self.config.clone(),
- self.encryption.clone(),
- ));
- Some(postgres)
- },
- });
- } else {
- self.restful_postgres.store(None);
- }
- }
-
- fn user_service(&self) -> Arc<dyn UserCloudService> {
- // handle the realtime collab update event.
- let (user_update_tx, user_update_rx) = tokio::sync::mpsc::channel(1);
-
- let collab_update_handler = Box::new(RealtimeCollabUpdateHandler::new(
- Arc::downgrade(&self.collab_update_sender),
- self.device_id.clone(),
- self.encryption.clone(),
- ));
-
- // handle the realtime user event.
- let user_handler = Box::new(RealtimeUserHandler(user_update_tx));
-
- let handlers: Vec<Box<dyn RealtimeEventHandler>> = vec![collab_update_handler, user_handler];
- Arc::new(SupabaseUserServiceImpl::new(
- SupabaseServerServiceImpl(self.restful_postgres.clone()),
- handlers,
- Some(user_update_rx),
- ))
- }
-
- fn folder_service(&self) -> Arc<dyn FolderCloudService> {
- Arc::new(SupabaseFolderServiceImpl::new(SupabaseServerServiceImpl(
- self.restful_postgres.clone(),
- )))
- }
-
- fn database_service(&self) -> Arc<dyn DatabaseCloudService> {
- Arc::new(SupabaseDatabaseServiceImpl::new(SupabaseServerServiceImpl(
- self.restful_postgres.clone(),
- )))
- }
-
- fn database_ai_service(&self) -> Option<Arc<dyn DatabaseAIService>> {
- None
- }
-
- fn document_service(&self) -> Arc<dyn DocumentCloudService> {
- Arc::new(SupabaseDocumentServiceImpl::new(SupabaseServerServiceImpl(
- self.restful_postgres.clone(),
- )))
- }
-
- fn collab_storage(&self, collab_object: &CollabObject) -> Option<Arc<dyn RemoteCollabStorage>> {
- let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
- self
- .collab_update_sender
- .insert(collab_object.object_id.clone(), tx);
-
- Some(Arc::new(SupabaseCollabStorageImpl::new(
- SupabaseServerServiceImpl(self.restful_postgres.clone()),
- Some(rx),
- self.encryption.clone(),
- )))
- }
-
- fn file_storage(&self) -> Option<Arc<dyn StorageCloudService>> {
- None
- }
-
- fn search_service(&self) -> Option<Arc<dyn SearchCloudService>> {
- None
- }
-}
diff --git a/frontend/rust-lib/flowy-sqlite-vec/Cargo.toml b/frontend/rust-lib/flowy-sqlite-vec/Cargo.toml
new file mode 100644
index 0000000000..58406eb047
--- /dev/null
+++ b/frontend/rust-lib/flowy-sqlite-vec/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "flowy-sqlite-vec"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+sqlite-vec = "0.1.6"
+rusqlite = { version = "0.35.0", features = ["bundled"] }
+rusqlite_migration = { version = "2.1.0", features = ["from-directory"] }
+include_dir = "0.7.4"
+anyhow.workspace = true
+tracing.workspace = true
+uuid.workspace = true
+zerocopy = "0.9.0-alpha.0"
+tokio.workspace = true
+flowy-ai-pub = { workspace = true }
+serde_json.workspace = true
+chrono.workspace = true
+
+[dev-dependencies]
+tempfile = "3.8.0"
diff --git a/frontend/rust-lib/flowy-sqlite-vec/migrations/001-init/up.sql b/frontend/rust-lib/flowy-sqlite-vec/migrations/001-init/up.sql
new file mode 100644
index 0000000000..9407d73c05
--- /dev/null
+++ b/frontend/rust-lib/flowy-sqlite-vec/migrations/001-init/up.sql
@@ -0,0 +1,25 @@
+CREATE VIRTUAL TABLE af_collab_embeddings
+USING vec0(
+ workspace_id TEXT NOT NULL,
+ object_id TEXT NOT NULL,
+ fragment_id TEXT NOT NULL,
+ content_type INTEGER NOT NULL,
+ content TEXT NOT NULL,
+ metadata TEXT,
+ fragment_index INTEGER NOT NULL DEFAULT 0,
+ embedder_type INTEGER NOT NULL DEFAULT 0,
+ embedding float[768]
+);
+
+CREATE TABLE af_pending_index_collab
+(
+ oid TEXT PRIMARY KEY NOT NULL,
+ workspace_id TEXT NOT NULL,
+ content TEXT NOT NULL,
+ collab_type SMALLINT NOT NULL,
+ updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ indexed_at TIMESTAMP DEFAULT NULL
+);
+
+-- create index for oid and workspace_id
+CREATE INDEX collab_table_oid_workspace_id_idx ON af_pending_index_collab (oid, workspace_id);
diff --git a/frontend/rust-lib/flowy-sqlite-vec/src/db.rs b/frontend/rust-lib/flowy-sqlite-vec/src/db.rs
new file mode 100644
index 0000000000..6efd2800e3
--- /dev/null
+++ b/frontend/rust-lib/flowy-sqlite-vec/src/db.rs
@@ -0,0 +1,352 @@
+use crate::entities::PendingIndexedCollab;
+use crate::init_sqlite_vector_extension;
+use crate::migration::init_sqlite_with_migrations;
+use anyhow::{Context, Result};
+use flowy_ai_pub::entities::{EmbeddedChunk, SearchResult};
+use rusqlite::{params, Connection, ToSql};
+use serde_json::Value;
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+use uuid::Uuid;
+use zerocopy::IntoBytes;
+
+pub struct VectorSqliteDB {
+ conn: Arc<Mutex<Connection>>,
+}
+
+unsafe impl Send for VectorSqliteDB {}
+unsafe impl Sync for VectorSqliteDB {}
+
+impl VectorSqliteDB {
+ pub fn new(root: PathBuf) -> Result<Self> {
+ init_sqlite_vector_extension();
+ let db_path = root.join("vector.db");
+ let conn = Arc::new(Mutex::new(init_sqlite_with_migrations(db_path.as_path())?));
+ Ok(Self { conn })
+ }
+
+ pub async fn select_collabs_fragment_ids(
+ &self,
+ object_ids: &[String],
+ ) -> Result<HashMap<Uuid, Vec<String>>> {
+ if object_ids.is_empty() {
+ return Ok(HashMap::new());
+ }
+
+ let placeholders = std::iter::repeat("?")
+ .take(object_ids.len())
+ .collect::<Vec<_>>()
+ .join(", ");
+
+ let sql = format!(
+ "SELECT fragment_id, object_id FROM af_collab_embeddings WHERE object_id IN ({})",
+ placeholders
+ );
+
+ let conn = self.conn.lock().await;
+ let mut stmt = conn
+ .prepare(&sql)
+ .context("Preparing select_collabs_fragment_ids")?;
+
+ let params: Vec<&dyn ToSql> = object_ids.iter().map(|s| s as &dyn ToSql).collect();
+ let mut rows = stmt
+ .query(params.as_slice())
+ .context("Executing select_collabs_fragment_ids")?;
+
+ let mut map: HashMap<Uuid, Vec<String>> = HashMap::new();
+ while let Some(row) = rows.next()? {
+ let fragment_id: String = row.get(0)?;
+ let oid_str: String = row.get(1)?;
+ let oid = Uuid::parse_str(&oid_str)
+ .map_err(|e| anyhow::anyhow!("Invalid UUID `{}` in DB: {}", oid_str, e))?;
+ map.entry(oid).or_default().push(fragment_id);
+ }
+
+ Ok(map)
+ }
+
+ pub async fn delete_collab(&self, workspace_id: &str, object_id: &str) -> Result<()> {
+ let mut conn = self.conn.lock().await;
+ let tx = conn
+ .transaction()
+ .context("Starting delete_collab transaction")?;
+ tx.execute(
+ "DELETE FROM af_collab_embeddings
+ WHERE workspace_id = ?1
+ AND object_id = ?2",
+ rusqlite::params![workspace_id, object_id],
+ )
+ .context("Deleting collab embeddings")?;
+
+ tx.commit()
+ .context("Committing delete_collab transaction")?;
+ Ok(())
+ }
+
+ /// Inserts or replaces all of `fragments` for the given (workspace_id, object_id),
+ /// deleting anything else in that scope first, and storing the new vector blobs.
+ pub async fn upsert_collabs_embeddings(
+ &self,
+ workspace_id: &str,
+ object_id: &str,
+ fragments: Vec<EmbeddedChunk>,
+ ) -> Result<()> {
+ if fragments.is_empty() {
+ return Ok(());
+ }
+
+ let mut conn = self.conn.lock().await;
+ let tx = conn.transaction().context("Starting transaction")?;
+
+ // 1) Collect new IDs
+ let new_ids: Vec<&str> = fragments.iter().map(|f| f.fragment_id.as_str()).collect();
+
+ // 2) Load existing IDs from the DB
+ let mut stmt = tx.prepare(
+ "SELECT fragment_id
+ FROM af_collab_embeddings
+ WHERE workspace_id = ?1
+ AND object_id = ?2",
+ )?;
+ let existing_ids: HashSet<String> = stmt
+ .query_map(params![workspace_id, object_id], |row| row.get(0))?
+ .collect::<Result<_, _>>()?;
+
+ // 3) Compute which to delete (existing − new)
+ let to_delete: Vec<&str> = existing_ids
+ .iter()
+ .filter_map(|id| {
+ if !new_ids.contains(&id.as_str()) {
+ Some(id.as_str())
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ // 4) Delete stale fragments (if any)
+ if !to_delete.is_empty() {
+ let placeholders = std::iter::repeat("?")
+ .take(to_delete.len())
+ .collect::<Vec<_>>()
+ .join(", ");
+ let sql = format!(
+ "DELETE FROM af_collab_embeddings
+ WHERE workspace_id = ?1
+ AND object_id = ?2
+ AND fragment_id IN ({})",
+ placeholders
+ );
+ let mut params: Vec<&dyn ToSql> = Vec::with_capacity(2 + to_delete.len());
+ params.push(&workspace_id);
+ params.push(&object_id);
+ params.extend(to_delete.iter().map(|s| s as &dyn ToSql));
+ tx.execute(&sql, params.as_slice())
+ .context("Deleting stale fragments")?;
+ }
+
+ // 5) Insert only the brand-new fragments (new − existing)
+ let to_insert: Vec<&EmbeddedChunk> = fragments
+ .iter()
+ .filter(|f| !existing_ids.contains(&f.fragment_id))
+ .collect();
+
+ if !to_insert.is_empty() {
+ let mut insert = tx.prepare(
+ "INSERT INTO af_collab_embeddings
+ (workspace_id, object_id, fragment_id,
+ content_type, content, metadata,
+ fragment_index, embedder_type, embedding)
+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
+ )?;
+
+ for frag in to_insert {
+ // skip if content missing
+ if frag.content.is_none() {
+ continue;
+ }
+ insert
+ .execute(rusqlite::params![
+ workspace_id,
+ object_id,
+ &frag.fragment_id,
+ frag.content_type,
+ frag.content.clone().unwrap_or_default(),
+ frag.metadata,
+ frag.fragment_index,
+ frag.embedder_type,
+ frag
+ .embeddings
+ .as_ref()
+ .map(|b| b.as_bytes())
+ .unwrap_or(&[]),
+ ])
+ .context("Inserting new fragment")?;
+ }
+ drop(insert);
+ }
+ drop(stmt);
+
+ tx.commit().context("Committing transaction")?;
+ Ok(())
+ }
+
+ pub async fn search(
+ &self,
+ workspace_id: &str,
+ query: &[f32],
+ top_k: i32,
+ ) -> Result<Vec<SearchResult>> {
+ self
+ .search_with_score(workspace_id, query, top_k, 0.4)
+ .await
+ }
+
+ pub async fn search_with_score(
+ &self,
+ workspace_id: &str,
+ query: &[f32],
+ top_k: i32,
+ min_score: f32,
+ ) -> Result<Vec<SearchResult>> {
+ // clamp min_score to [0,1]
+ let min_score = min_score.clamp(0.0, 1.0);
+ // distance = 1 - score, so we only want distance <= max_distance
+ let max_distance = 1.0 - min_score;
+ let query_blob = query.as_bytes();
+
+ // k-NN MATCH with embedded distance filter, compute score = 1-distance
+ let sql = "\
+ SELECT
+ object_id,
+ content,
+ metadata,
+ distance,
+ (1.0 - distance) AS score
+ FROM af_collab_embeddings
+ WHERE embedding MATCH ?
+ AND k = ?
+ AND workspace_id = ?
+ AND distance <= ?
+ ORDER BY distance ASC
+ ";
+
+ let conn = self.conn.lock().await;
+ let mut stmt = conn.prepare(sql)?;
+ let mut rows = stmt.query(params![
+ query_blob, // MATCH
+ top_k, // number of neighbors
+ workspace_id, // workspace filter
+ max_distance, // only distances ≤ this
+ ])?;
+
+ let mut out = Vec::new();
+ while let Some(row) = rows.next()? {
+ let oid_str: String = row.get(0)?;
+ let oid = match Uuid::parse_str(&oid_str) {
+ Ok(u) => u,
+ Err(_) => continue,
+ };
+ let content: String = row.get(1)?;
+ let metadata = row
+ .get::<_, Option<String>>(2)?
+ .and_then(|s| serde_json::from_str::<Value>(&s).ok());
+ let score: f32 = row.get(4)?;
+
+ out.push(SearchResult {
+ oid,
+ content,
+ metadata,
+ score,
+ });
+ }
+
+ Ok(out)
+ }
+
+ pub async fn delete_pending_indexed_collab(
+ &self,
+ workspace_id: &str,
+ object_ids: Vec<String>,
+ ) -> Result<()> {
+ // Nothing to do if no object_ids provided
+ if object_ids.is_empty() {
+ return Ok(());
+ }
+
+ // Lock and begin a transaction
+ let mut conn = self.conn.lock().await;
+ let tx = conn
+ .transaction()
+ .context("Starting delete_pending_indexed_collab transaction")?;
+
+ // Build the `IN (?, ?, …)` clause dynamically
+ let placeholders = std::iter::repeat("?")
+ .take(object_ids.len())
+ .collect::<Vec<_>>()
+ .join(", ");
+ let sql = format!(
+ "DELETE FROM af_pending_index_collab WHERE workspace_id = ?1 AND object_id IN ({})",
+ placeholders
+ );
+
+ // Bind workspace_id as param 1, then each object_id
+ let mut query_params: Vec<&dyn ToSql> = Vec::with_capacity(1 + object_ids.len());
+ query_params.push(&workspace_id);
+ for oid in &object_ids {
+ query_params.push(oid as &dyn ToSql);
+ }
+
+ // Execute and commit
+ tx.execute(&sql, query_params.as_slice())
+ .context("Deleting pending indexed collabs")?;
+ tx.commit()
+ .context("Committing delete_pending_indexed_collab transaction")?;
+
+ Ok(())
+ }
+
+ pub async fn queue_pending_indexed_collab(&self, data: PendingIndexedCollab) -> Result<()> {
+ self.batch_queue_pending_indexed_collabs(vec![data]).await
+ }
+
+ /// Enqueue multiple pending collabs in a single transaction.
+ pub async fn batch_queue_pending_indexed_collabs(
+ &self,
+ data: Vec<PendingIndexedCollab>,
+ ) -> Result<()> {
+ if data.is_empty() {
+ return Ok(());
+ }
+
+ // lock once, start one transaction
+ let mut conn = self.conn.lock().await;
+ let tx = conn.transaction().context("Starting transaction")?;
+
+ // prepare INSERT only once
+ let mut stmt = tx.prepare(
+ "INSERT INTO af_pending_index_collab
+ (workspace_id, object_id, collab_type, content)
+ VALUES (?1, ?2, ?3, ?4)",
+ )?;
+
+ // execute for each item
+ for item in data {
+ stmt
+ .execute(rusqlite::params![
+ item.workspace_id,
+ item.object_id,
+ item.collab_type,
+ item.content,
+ ])
+ .context("Inserting pending collab")?;
+ }
+
+ // drop the Statement (releases borrow on tx) then commit
+ drop(stmt);
+ tx.commit().context("Committing transaction")?;
+ Ok(())
+ }
+}
diff --git a/frontend/rust-lib/flowy-sqlite-vec/src/entities.rs b/frontend/rust-lib/flowy-sqlite-vec/src/entities.rs
new file mode 100644
index 0000000000..bbd39d342b
--- /dev/null
+++ b/frontend/rust-lib/flowy-sqlite-vec/src/entities.rs
@@ -0,0 +1,7 @@
+#[derive(Clone, Debug)]
+pub struct PendingIndexedCollab {
+ pub object_id: String,
+ pub workspace_id: String,
+ pub content: String,
+ pub collab_type: i16,
+}
diff --git a/frontend/rust-lib/flowy-sqlite-vec/src/lib.rs b/frontend/rust-lib/flowy-sqlite-vec/src/lib.rs
new file mode 100644
index 0000000000..ffdd97cb6f
--- /dev/null
+++ b/frontend/rust-lib/flowy-sqlite-vec/src/lib.rs
@@ -0,0 +1,13 @@
+use rusqlite::ffi::sqlite3_auto_extension;
+use sqlite_vec::sqlite3_vec_init;
+
+pub mod db;
+pub mod entities;
+mod migration;
+
+#[allow(clippy::missing_transmute_annotations)]
+pub fn init_sqlite_vector_extension() {
+ unsafe {
+ sqlite3_auto_extension(Some(std::mem::transmute(sqlite3_vec_init as *const ())));
+ }
+}
diff --git a/frontend/rust-lib/flowy-sqlite-vec/src/migration.rs b/frontend/rust-lib/flowy-sqlite-vec/src/migration.rs
new file mode 100644
index 0000000000..53c0eed57e
--- /dev/null
+++ b/frontend/rust-lib/flowy-sqlite-vec/src/migration.rs
@@ -0,0 +1,16 @@
+use include_dir::{include_dir, Dir};
+use rusqlite::Connection;
+use rusqlite_migration::Migrations;
+use std::path::Path;
+use std::sync::LazyLock;
+
+static MIGRATIONS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/migrations");
+static MIGRATIONS: LazyLock<Migrations<'static>> =
+ LazyLock::new(|| Migrations::from_directory(&MIGRATIONS_DIR).unwrap());
+
+/// Initialize a new SQLite database with migrations
+pub fn init_sqlite_with_migrations(db_path: &Path) -> anyhow::Result<Connection> {
+ let mut conn = Connection::open(db_path)?;
+ MIGRATIONS.to_latest(&mut conn)?;
+ Ok(conn)
+}
diff --git a/frontend/rust-lib/flowy-sqlite-vec/tests/main.rs b/frontend/rust-lib/flowy-sqlite-vec/tests/main.rs
new file mode 100644
index 0000000000..ffdd317850
--- /dev/null
+++ b/frontend/rust-lib/flowy-sqlite-vec/tests/main.rs
@@ -0,0 +1,308 @@
+use anyhow::Result;
+use flowy_ai_pub::entities::EmbeddedChunk;
+use flowy_sqlite_vec::db::VectorSqliteDB;
+use flowy_sqlite_vec::init_sqlite_vector_extension;
+use std::collections::HashSet;
+use tempfile::tempdir;
+use uuid::Uuid;
+
+#[tokio::test]
+async fn test_vector_sqlite_db_basic_operations() -> Result<()> {
+ // Initialize sqlite-vec extension
+ init_sqlite_vector_extension();
+
+ // Create a temporary directory for the test database
+ let temp_dir = tempdir()?;
+
+ // Create the VectorSqliteDB
+ let db = VectorSqliteDB::new(temp_dir.into_path())?;
+
+ // Test inserting vector embeddings
+ let oid = Uuid::new_v4().to_string();
+ let fragments = vec![
+ create_test_fragment(&oid, 0, generate_embedding_with_size(768, 0.1)),
+ create_test_fragment(&oid, 1, generate_embedding_with_size(768, 0.2)),
+ create_test_fragment(&oid, 2, generate_embedding_with_size(768, 0.3)),
+ ];
+ let workspace_id = Uuid::new_v4();
+ db.upsert_collabs_embeddings(&workspace_id.to_string(), &oid, fragments)
+ .await?;
+
+ // Test querying fragment IDs
+ let result = db.select_collabs_fragment_ids(&[oid.clone()]).await?;
+ assert_eq!(result.len(), 1);
+ assert!(result.contains_key(&Uuid::parse_str(&oid)?));
+ assert_eq!(result.get(&Uuid::parse_str(&oid)?).unwrap().len(), 3);
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_upsert_and_remove_fragments() -> Result<()> {
+ // Initialize sqlite-vec extension
+ init_sqlite_vector_extension();
+
+ // Create a temporary directory for the test database
+ let temp_dir = tempdir()?;
+
+ // Create the VectorSqliteDB
+ let db = VectorSqliteDB::new(temp_dir.into_path())?;
+
+ // Test inserting initial vector embeddings
+ let oid = Uuid::new_v4().to_string();
+ let initial_fragments = vec![
+ create_test_fragment(&oid, 0, generate_embedding_with_size(768, 0.1)),
+ create_test_fragment(&oid, 1, generate_embedding_with_size(768, 0.2)),
+ create_test_fragment(&oid, 2, generate_embedding_with_size(768, 0.3)),
+ ];
+
+ let workspace_id = Uuid::new_v4();
+ db.upsert_collabs_embeddings(&workspace_id.to_string(), &oid, initial_fragments)
+ .await?;
+
+ // Verify initial fragments
+ let result = db.select_collabs_fragment_ids(&[oid.clone()]).await?;
+ assert_eq!(result.get(&Uuid::parse_str(&oid)?).unwrap().len(), 3);
+
+ // Update with a subset of fragments (this should remove the missing one)
+ let updated_fragments = vec![
+ create_test_fragment(&oid, 0, generate_embedding_with_size(768, 0.1)),
+ create_test_fragment(&oid, 2, generate_embedding_with_size(768, 0.3)),
+ ];
+
+ db.upsert_collabs_embeddings(&workspace_id.to_string(), &oid, updated_fragments)
+ .await?;
+ // Verify fragment count is now 2
+ let result = db.select_collabs_fragment_ids(&[oid.clone()]).await?;
+ assert_eq!(result.get(&Uuid::parse_str(&oid)?).unwrap().len(), 2);
+
+ let result = db
+ .search(
+ &workspace_id.to_string(),
+ &generate_embedding_with_size(768, 0.1),
+ 1,
+ )
+ .await
+ .unwrap();
+ assert!(!result.is_empty());
+ assert_eq!(result[0].oid, Uuid::parse_str(&oid).unwrap());
+ assert_eq!(result[0].content, "Content for fragment 0".to_string());
+ dbg!(result);
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_empty_fragments_noop_and_select_empty() -> Result<()> {
+ init_sqlite_vector_extension();
+ let temp_dir = tempdir()?;
+ let db = VectorSqliteDB::new(temp_dir.into_path())?;
+
+ let oid = Uuid::new_v4().to_string();
+ let workspace_id = Uuid::new_v4().to_string();
+
+ // Upsert with an empty fragments Vec should not error and not insert anything
+ db.upsert_collabs_embeddings(&workspace_id, &oid, Vec::new())
+ .await?;
+
+ // select_collabs_fragment_ids should return an empty map
+ let result = db.select_collabs_fragment_ids(&[oid.clone()]).await?;
+ assert!(
+ result.is_empty(),
+ "Expected no fragments stored, got {:?}",
+ result
+ );
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_duplicate_upsert_idempotent() -> Result<()> {
+ init_sqlite_vector_extension();
+ let temp_dir = tempdir()?;
+ let db = VectorSqliteDB::new(temp_dir.into_path())?;
+
+ let oid = Uuid::new_v4().to_string();
+ let workspace_id = Uuid::new_v4().to_string();
+ let fragments = vec![
+ create_test_fragment(&oid, 0, generate_embedding_with_size(768, 0.5)),
+ create_test_fragment(&oid, 1, generate_embedding_with_size(768, 0.6)),
+ ];
+
+ // First upsert
+ db.upsert_collabs_embeddings(&workspace_id, &oid, fragments.clone())
+ .await?;
+ let first = db.select_collabs_fragment_ids(&[oid.clone()]).await?;
+ let set1: HashSet<_> = first
+ .get(&Uuid::parse_str(&oid)?)
+ .unwrap()
+ .clone()
+ .into_iter()
+ .collect();
+
+ // Second upsert with the exact same fragments
+ db.upsert_collabs_embeddings(&workspace_id, &oid, fragments)
+ .await?;
+ let second = db.select_collabs_fragment_ids(&[oid.clone()]).await?;
+ let set2: HashSet<_> = second
+ .get(&Uuid::parse_str(&oid)?)
+ .unwrap()
+ .clone()
+ .into_iter()
+ .collect();
+
+ assert_eq!(
+ set1, set2,
+ "Upserting the same fragments should be idempotent"
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_search_no_hits() -> Result<()> {
+ init_sqlite_vector_extension();
+ let temp_dir = tempdir()?;
+ let db = VectorSqliteDB::new(temp_dir.into_path())?;
+
+ let oid = Uuid::new_v4().to_string();
+ let workspace_id = Uuid::new_v4().to_string();
+ // Insert a single fragment at vector [1.0,...]
+ let frags = vec![create_test_fragment(
+ &oid,
+ 0,
+ generate_embedding_with_size(768, 1.0),
+ )];
+ db.upsert_collabs_embeddings(&workspace_id, &oid, frags)
+ .await?;
+
+ // Query with a very different vector should return empty
+ let query = generate_embedding_with_size(768, -1.0);
+ let results = db.search(&workspace_id, &query, 1).await?;
+ assert!(
+ results.is_empty(),
+ "Expected no near neighbors for orthogonal vector"
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_multi_workspace_isolation() -> Result<()> {
+ init_sqlite_vector_extension();
+ let temp_dir = tempdir()?;
+ let db = VectorSqliteDB::new(temp_dir.into_path())?;
+
+ let oid = Uuid::new_v4().to_string();
+ let ws1 = Uuid::new_v4().to_string();
+ let ws2 = Uuid::new_v4().to_string();
+
+ // Insert identical fragment into two workspaces but with different embeddings
+ let frag = create_test_fragment(&oid, 0, generate_embedding_with_size(768, 0.9));
+ db.upsert_collabs_embeddings(&ws1, &oid, vec![frag.clone()])
+ .await?;
+ let frag2 = create_test_fragment(&oid, 0, generate_embedding_with_size(768, -0.9));
+ db.upsert_collabs_embeddings(&ws2, &oid, vec![frag2.clone()])
+ .await?;
+
+ // Searching in ws1 should not return ws2's fragment
+ let res1 = db
+ .search(&ws1, &generate_embedding_with_size(768, 0.9), 1)
+ .await?;
+ assert_eq!(res1.len(), 1);
+ assert_eq!(res1[0].oid, Uuid::parse_str(&oid)?);
+
+ // Searching in ws2 should not return ws1's fragment
+ let res2 = db
+ .search(&ws2, &generate_embedding_with_size(768, -0.9), 1)
+ .await?;
+ assert_eq!(res2.len(), 1);
+ assert_eq!(res2[0].oid, Uuid::parse_str(&oid)?);
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_select_multiple_oids() -> Result<()> {
+ init_sqlite_vector_extension();
+ let temp_dir = tempdir()?;
+ let db = VectorSqliteDB::new(temp_dir.into_path())?;
+
+ let ws = Uuid::new_v4().to_string();
+ let oid1 = Uuid::new_v4().to_string();
+ let oid2 = Uuid::new_v4().to_string();
+
+ db.upsert_collabs_embeddings(
+ &ws,
+ &oid1,
+ vec![create_test_fragment(
+ &oid1,
+ 0,
+ generate_embedding_with_size(768, 0.1),
+ )],
+ )
+ .await?;
+ db.upsert_collabs_embeddings(
+ &ws,
+ &oid2,
+ vec![create_test_fragment(
+ &oid2,
+ 0,
+ generate_embedding_with_size(768, 0.2),
+ )],
+ )
+ .await?;
+
+ let map = db
+ .select_collabs_fragment_ids(&[oid1.clone(), oid2.clone()])
+ .await?;
+ assert_eq!(map.len(), 2);
+ assert!(map.contains_key(&Uuid::parse_str(&oid1)?));
+ assert!(map.contains_key(&Uuid::parse_str(&oid2)?));
+ assert_eq!(map[&Uuid::parse_str(&oid1)?].len(), 1);
+ assert_eq!(map[&Uuid::parse_str(&oid2)?].len(), 1);
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_skip_missing_content() -> Result<()> {
+ init_sqlite_vector_extension();
+ let temp_dir = tempdir()?;
+ let db = VectorSqliteDB::new(temp_dir.into_path())?;
+
+ let ws = Uuid::new_v4().to_string();
+ let oid = Uuid::new_v4().to_string();
+
+ // One fragment with no content (should be skipped), one with content
+ let mut bad = create_test_fragment(&oid, 0, generate_embedding_with_size(768, 0.1));
+ bad.content = None;
+ let good = create_test_fragment(&oid, 1, generate_embedding_with_size(768, 0.2));
+
+ db.upsert_collabs_embeddings(&ws, &oid, vec![bad, good.clone()])
+ .await?;
+
+ let map = db.select_collabs_fragment_ids(&[oid.clone()]).await?;
+ let frags = &map[&Uuid::parse_str(&oid)?];
+ assert_eq!(frags.len(), 1);
+ assert_eq!(frags[0], good.fragment_id);
+
+ Ok(())
+}
+
+fn generate_embedding_with_size(size: usize, value: f32) -> Vec<f32> {
+ vec![value; size]
+}
+
+fn create_test_fragment(oid: &str, index: i32, embeddings: Vec<f32>) -> EmbeddedChunk {
+ let fragment_id = format!("fragment_{}", index);
+
+ EmbeddedChunk {
+ fragment_id,
+ object_id: oid.to_string(),
+ content_type: 1,
+ content: Some(format!("Content for fragment {}", index)),
+ metadata: Some(format!("Metadata for fragment {}", index)),
+ fragment_index: index,
+ embedder_type: 1,
+ embeddings: Some(embeddings),
+ }
+}
diff --git a/frontend/rust-lib/flowy-sqlite/Cargo.toml b/frontend/rust-lib/flowy-sqlite/Cargo.toml
index 345b05f903..1207777281 100644
--- a/frontend/rust-lib/flowy-sqlite/Cargo.toml
+++ b/frontend/rust-lib/flowy-sqlite/Cargo.toml
@@ -15,7 +15,7 @@ serde_json.workspace = true
anyhow.workspace = true
r2d2 = "0.8.10"
-libsqlite3-sys = { version = "0.27.0", features = ["bundled"] }
+libsqlite3-sys = { version = "0.33.0", features = ["bundled"] }
scheduled-thread-pool = "0.2.6"
openssl = { version = "0.10.62", optional = true, features = ["vendored"] }
openssl-sys = { version = "0.9.98", optional = true, features = ["vendored"] }
diff --git a/frontend/rust-lib/flowy-sqlite/migrations/2025-04-25-071459_local_ai_model/down.sql b/frontend/rust-lib/flowy-sqlite/migrations/2025-04-25-071459_local_ai_model/down.sql
index d9a93fe9a1..39cfc39f80 100644
--- a/frontend/rust-lib/flowy-sqlite/migrations/2025-04-25-071459_local_ai_model/down.sql
+++ b/frontend/rust-lib/flowy-sqlite/migrations/2025-04-25-071459_local_ai_model/down.sql
@@ -1 +1,2 @@
-- This file should undo anything in `up.sql`
+DROP TABLE local_ai_model_table;
\ No newline at end of file
diff --git a/frontend/rust-lib/flowy-sqlite/migrations/2025-04-28-070644_collab_table/down.sql b/frontend/rust-lib/flowy-sqlite/migrations/2025-04-28-070644_collab_table/down.sql
new file mode 100644
index 0000000000..e5862bcb05
--- /dev/null
+++ b/frontend/rust-lib/flowy-sqlite/migrations/2025-04-28-070644_collab_table/down.sql
@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE collab_table;
\ No newline at end of file
diff --git a/frontend/rust-lib/flowy-sqlite/migrations/2025-04-28-070644_collab_table/up.sql b/frontend/rust-lib/flowy-sqlite/migrations/2025-04-28-070644_collab_table/up.sql
new file mode 100644
index 0000000000..ee005ed69b
--- /dev/null
+++ b/frontend/rust-lib/flowy-sqlite/migrations/2025-04-28-070644_collab_table/up.sql
@@ -0,0 +1,10 @@
+-- Your SQL goes here
+CREATE TABLE index_collab_record_table
+(
+ oid TEXT PRIMARY KEY NOT NULL,
+ workspace_id TEXT NOT NULL,
+ content_hash TEXT NOT NULL
+);
+
+-- create index for workspace_id
+CREATE INDEX collab_record_table_workspace_id_idx ON index_collab_record_table (workspace_id);
diff --git a/frontend/rust-lib/flowy-sqlite/src/lib.rs b/frontend/rust-lib/flowy-sqlite/src/lib.rs
index 0911b48fd2..c8acc7b52e 100644
--- a/frontend/rust-lib/flowy-sqlite/src/lib.rs
+++ b/frontend/rust-lib/flowy-sqlite/src/lib.rs
@@ -18,9 +18,6 @@ mod sqlite_impl;
pub mod schema;
-#[macro_use]
-pub mod macros;
-
pub type Error = diesel::result::Error;
pub mod prelude {
pub use diesel::SqliteConnection;
diff --git a/frontend/rust-lib/flowy-sqlite/src/macros.rs b/frontend/rust-lib/flowy-sqlite/src/macros.rs
deleted file mode 100644
index ad6d3b377a..0000000000
--- a/frontend/rust-lib/flowy-sqlite/src/macros.rs
+++ /dev/null
@@ -1,220 +0,0 @@
-#[rustfmt::skip]
-/*
-diesel master support on_conflict on sqlite but not 1.4.7 version. Workaround for this
-
-match dsl::workspace_table
- .filter(workspace_table::id.eq(table.id.clone()))
- .count()
- .get_result(conn)
- .unwrap_or(0)
-{
- 0 => diesel::insert_into(workspace_table::table).values(table)
- .on_conflict(workspace_table::id)
- .do_update()
- .set(WorkspaceTableChangeset::from_table(workspace_table))
- .execute(conn)?,
- _ => {
- let changeset = WorkspaceTableChangeset::from_table(table);
- let filter = dsl::workspace_table.filter(workspace_table::id.eq(changeset.id.clone()));
- diesel::update(filter).set(changeset).execute(conn)?;
- },
-}
-
-is equivalent to:
-
-match diesel_record_count!(workspace_table, &table.id, conn) {
- 0 => diesel_insert_table!(workspace_table, table, conn),
- _ => diesel_update_table!(workspace_table, WorkspaceTableChangeset::from_table(table), &*conn),
-}
-*/
-
-#[macro_export]
-macro_rules! diesel_insert_table {
- (
- $table_name:ident,
- $table:expr,
- $connection:expr
- ) => {
- {
- let _ = diesel::insert_into($table_name::table)
- .values($table)
- // .on_conflict($table_name::dsl::id)
- // .do_update()
- // .set(WorkspaceTableChangeset::from_table(workspace_table))
- .execute($connection)?;
- }
- };
-}
-
-#[macro_export]
-macro_rules! diesel_record_count {
- (
- $table_name:ident,
- $id:expr,
- $connection:expr
- ) => {
- $table_name::dsl::$table_name
- .filter($table_name::dsl::id.eq($id.clone()))
- .count()
- .get_result($connection)
- .unwrap_or(0);
- };
-}
-
-#[macro_export]
-macro_rules! diesel_revision_record_count {
- (
- $table_name:expr,
- $filter:expr,
- $connection:expr
- ) => {
- $table_name
- .filter($table_name::dsl::id.eq($id))
- .count()
- .get_result($connection)
- .unwrap_or(0);
- };
-}
-
-#[macro_export]
-macro_rules! diesel_update_table {
- (
- $table_name:ident,
- $changeset:expr,
- $connection:expr
- ) => {{
- let filter =
- $table_name::dsl::$table_name.filter($table_name::dsl::id.eq($changeset.id.clone()));
- let affected_row = diesel::update(filter)
- .set($changeset)
- .execute($connection)?;
- debug_assert_eq!(affected_row, 1);
- }};
-}
-
-#[macro_export]
-macro_rules! diesel_delete_table {
- (
- $table_name:ident,
- $id:ident,
- $connection:ident
- ) => {
- let filter = $table_name::dsl::$table_name.filter($table_name::dsl::id.eq($id));
- let affected_row = diesel::delete(filter).execute(&*$connection)?;
- debug_assert_eq!(affected_row, 1);
- };
-}
-
-#[macro_export]
-macro_rules! impl_sql_binary_expression {
- ($target:ident) => {
- impl diesel::serialize::ToSql<diesel::sql_types::Binary, diesel::sqlite::Sqlite> for $target {
- fn to_sql<W: std::io::Write>(
- &self,
- out: &mut diesel::serialize::Output<W, diesel::sqlite::Sqlite>,
- ) -> diesel::serialize::Result {
- let bytes: Vec<u8> = self.try_into().map_err(|e| format!("{:?}", e))?;
- diesel::serialize::ToSql::<diesel::sql_types::Binary, diesel::sqlite::Sqlite>::to_sql(
- &bytes, out,
- )
- }
- }
- // https://docs.diesel.rs/src/diesel/sqlite/types/mod.rs.html#30-33
- // impl FromSql<sql_types::Binary, Sqlite> for *const [u8] {
- // fn from_sql(bytes: Option<&SqliteValue>) -> deserialize::Result<Self> {
- // let bytes = not_none!(bytes).read_blob();
- // Ok(bytes as *const _)
- // }
- // }
- impl<DB> diesel::deserialize::FromSql<diesel::sql_types::Binary, DB> for $target
- where
- DB: diesel::backend::Backend,
- *const [u8]: diesel::deserialize::FromSql<diesel::sql_types::Binary, DB>,
- {
- fn from_sql(bytes: Option<&DB::RawValue>) -> diesel::deserialize::Result<Self> {
- let slice_ptr = <*const [u8] as diesel::deserialize::FromSql<
- diesel::sql_types::Binary,
- DB,
- >>::from_sql(bytes)?;
- let bytes = unsafe { &*slice_ptr };
-
- match $target::try_from(bytes) {
- Ok(object) => Ok(object),
- Err(e) => {
- log::error!(
- "{:?} deserialize from bytes fail. {:?}",
- std::any::type_name::<$target>(),
- e
- );
- panic!();
- },
- }
- }
- }
- };
-}
-
-#[macro_export]
-macro_rules! impl_sql_integer_expression {
- ($target:ident) => {
- impl<DB> diesel::serialize::ToSql<Integer, DB> for $target
- where
- DB: diesel::backend::Backend,
- i32: diesel::serialize::ToSql<Integer, DB>,
- {
- fn to_sql<W: std::io::Write>(
- &self,
- out: &mut diesel::serialize::Output<W, DB>,
- ) -> diesel::serialize::Result {
- (*self as i32).to_sql(out)
- }
- }
-
- impl<DB> diesel::deserialize::FromSql<Integer, DB> for $target
- where
- DB: diesel::backend::Backend,
- i32: diesel::deserialize::FromSql<Integer, DB>,
- {
- fn from_sql(bytes: Option<&DB::RawValue>) -> diesel::deserialize::Result<Self> {
- let smaill_int = i32::from_sql(bytes)?;
- Ok($target::from(smaill_int))
- }
- }
- };
-}
-
-#[macro_export]
-macro_rules! impl_rev_state_map {
- ($target:ident) => {
- impl std::convert::From<i32> for $target {
- fn from(value: i32) -> Self {
- match value {
- 0 => $target::Sync,
- 1 => $target::Ack,
- o => {
- tracing::error!("Unsupported rev state {}, fallback to RevState::Local", o);
- $target::Sync
- },
- }
- }
- }
-
- impl std::convert::From<$target> for RevisionState {
- fn from(s: $target) -> Self {
- match s {
- $target::Sync => RevisionState::Sync,
- $target::Ack => RevisionState::Ack,
- }
- }
- }
-
- impl std::convert::From<RevisionState> for $target {
- fn from(s: RevisionState) -> Self {
- match s {
- RevisionState::Sync => $target::Sync,
- RevisionState::Ack => $target::Ack,
- }
- }
- }
- };
-}
diff --git a/frontend/rust-lib/flowy-sqlite/src/schema.rs b/frontend/rust-lib/flowy-sqlite/src/schema.rs
index 1271589da6..290f66730b 100644
--- a/frontend/rust-lib/flowy-sqlite/src/schema.rs
+++ b/frontend/rust-lib/flowy-sqlite/src/schema.rs
@@ -54,6 +54,14 @@ diesel::table! {
}
}
+diesel::table! {
+ index_collab_record_table (oid) {
+ oid -> Text,
+ workspace_id -> Text,
+ content_hash -> Text,
+ }
+}
+
diesel::table! {
local_ai_model_table (name) {
name -> Text,
@@ -145,6 +153,7 @@ diesel::allow_tables_to_appear_in_same_query!(
chat_message_table,
chat_table,
collab_snapshot,
+ index_collab_record_table,
local_ai_model_table,
upload_file_part,
upload_file_table,
diff --git a/frontend/rust-lib/flowy-storage-pub/src/chunked_byte.rs b/frontend/rust-lib/flowy-storage-pub/src/chunked_byte.rs
index e5a78e2974..f8c4562815 100644
--- a/frontend/rust-lib/flowy-storage-pub/src/chunked_byte.rs
+++ b/frontend/rust-lib/flowy-storage-pub/src/chunked_byte.rs
@@ -84,7 +84,7 @@ impl ChunkedBytes {
/// Get the total number of chunks in the file.
pub fn total_chunks(&self) -> usize {
- ((self.file_size + self.chunk_size as u64 - 1) / self.chunk_size as u64) as usize
+ self.file_size.div_ceil(self.chunk_size as u64) as usize
}
/// Get the current offset in the file.
diff --git a/frontend/rust-lib/flowy-user-pub/src/workspace_service.rs b/frontend/rust-lib/flowy-user-pub/src/workspace_service.rs
index 84185d310f..f3735daf1b 100644
--- a/frontend/rust-lib/flowy-user-pub/src/workspace_service.rs
+++ b/frontend/rust-lib/flowy-user-pub/src/workspace_service.rs
@@ -3,7 +3,6 @@ use flowy_error::FlowyResult;
use flowy_folder_pub::entities::ImportFrom;
use lib_infra::async_trait::async_trait;
use std::collections::HashMap;
-use uuid::Uuid;
#[async_trait]
pub trait UserWorkspaceService: Send + Sync {
@@ -18,7 +17,4 @@ pub trait UserWorkspaceService: Send + Sync {
&self,
ids_by_database_id: HashMap<String, Vec<String>>,
) -> FlowyResult<()>;
-
- /// Removes local indexes when a workspace is left/deleted
- async fn did_delete_workspace(&self, workspace_id: &Uuid) -> FlowyResult<()>;
}
diff --git a/frontend/rust-lib/flowy-user/src/event_handler.rs b/frontend/rust-lib/flowy-user/src/event_handler.rs
index 4f9c9c22c3..943d05727b 100644
--- a/frontend/rust-lib/flowy-user/src/event_handler.rs
+++ b/frontend/rust-lib/flowy-user/src/event_handler.rs
@@ -478,7 +478,7 @@ pub async fn update_network_state_handler(
let reachable = data.into_inner().ty.is_reachable();
manager.cloud_service()?.set_network_reachable(reachable);
manager
- .user_status_callback
+ .app_life_cycle
.read()
.await
.on_network_status_changed(reachable);
diff --git a/frontend/rust-lib/flowy-user/src/event_map.rs b/frontend/rust-lib/flowy-user/src/event_map.rs
index 8c74eb312d..c7353a0b0c 100644
--- a/frontend/rust-lib/flowy-user/src/event_map.rs
+++ b/frontend/rust-lib/flowy-user/src/event_map.rs
@@ -10,6 +10,7 @@ use strum_macros::Display;
use uuid::Uuid;
use crate::event_handler::*;
+use crate::services::entities::{UserConfig, UserPaths};
use crate::user_manager::UserManager;
#[rustfmt::skip]
@@ -276,17 +277,15 @@ pub enum UserEvent {
}
#[async_trait]
-pub trait UserStatusCallback: Send + Sync + 'static {
- /// When the [AuthType] changed, this method will be called. Currently, the auth type
- /// will be changed when the user sign in or sign up.
- fn on_auth_type_changed(&self, _authenticator: AuthType) {}
+pub trait AppLifeCycle: Send + Sync + 'static {
/// Fires on app launch, but only if the user is already signed in.
async fn on_launch_if_authenticated(
&self,
_user_id: i64,
_cloud_config: &Option<UserCloudConfig>,
_workspace_id: &Uuid,
- _device_id: &str,
+ _user_config: &UserConfig,
+ _user_path: &UserPaths,
_workspace_type: &WorkspaceType,
) -> FlowyResult<()> {
Ok(())
@@ -301,7 +300,8 @@ pub trait UserStatusCallback: Send + Sync + 'static {
&self,
_user_id: i64,
_workspace_id: &Uuid,
- _device_id: &str,
+ _user_config: &UserConfig,
+ _user_path: &UserPaths,
_workspace_type: &WorkspaceType,
) -> FlowyResult<()> {
Ok(())
@@ -313,7 +313,8 @@ pub trait UserStatusCallback: Send + Sync + 'static {
_is_new_user: bool,
_user_profile: &UserProfile,
_workspace_id: &Uuid,
- _device_id: &str,
+ _user_config: &UserConfig,
+ _user_path: &UserPaths,
_workspace_type: &WorkspaceType,
) -> FlowyResult<()> {
Ok(())
@@ -324,6 +325,10 @@ pub trait UserStatusCallback: Send + Sync + 'static {
Ok(())
}
+ async fn on_workspace_closed(&self, _workspace_id: &Uuid) -> FlowyResult<()> {
+ Ok(())
+ }
+
/// Fires when a workspace is opened by the user.
async fn on_workspace_opened(
&self,
@@ -331,14 +336,24 @@ pub trait UserStatusCallback: Send + Sync + 'static {
_workspace_id: &Uuid,
_user_workspace: &UserWorkspace,
_workspace_type: &WorkspaceType,
+ _user_config: &UserConfig,
+ _user_path: &UserPaths,
) -> FlowyResult<()> {
Ok(())
}
+
+ async fn on_workspace_deleted(&self, _user_id: i64, _workspace_id: &Uuid) -> FlowyResult<()> {
+ Ok(())
+ }
fn on_network_status_changed(&self, _reachable: bool) {}
fn on_subscription_plans_updated(&self, _plans: Vec<SubscriptionPlan>) {}
fn on_storage_permission_updated(&self, _can_write: bool) {}
+
+ fn subscribe_full_indexed_finish(&self) -> Option<tokio::sync::watch::Receiver<bool>> {
+ None
+ }
}
-/// Acts as a placeholder [UserStatusCallback] for the user session, but does not perform any function
+/// Acts as a placeholder [AppLifeCycle] for the user session, but does not perform any function
pub(crate) struct DefaultUserStatusCallback;
-impl UserStatusCallback for DefaultUserStatusCallback {}
+impl AppLifeCycle for DefaultUserStatusCallback {}
diff --git a/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs b/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs
index 0c5e730008..690d9fb344 100644
--- a/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs
+++ b/frontend/rust-lib/flowy-user/src/services/authenticate_user.rs
@@ -89,7 +89,7 @@ impl AuthenticateUser {
pub fn get_index_path(&self) -> FlowyResult<PathBuf> {
let uid = self.user_id()?;
- Ok(PathBuf::from(self.user_paths.user_data_dir(uid)).join("indexes"))
+ Ok(self.user_paths.tanvity_index_path(uid))
}
pub fn get_user_data_dir(&self) -> FlowyResult<PathBuf> {
diff --git a/frontend/rust-lib/flowy-user/src/services/entities.rs b/frontend/rust-lib/flowy-user/src/services/entities.rs
index 4e034b3bdb..e2609f9d67 100644
--- a/frontend/rust-lib/flowy-user/src/services/entities.rs
+++ b/frontend/rust-lib/flowy-user/src/services/entities.rs
@@ -64,6 +64,10 @@ impl UserPaths {
format!("{}/{}", self.root, uid)
}
+ pub fn tanvity_index_path(&self, uid: i64) -> PathBuf {
+ PathBuf::from(self.user_data_dir(uid)).join("indexes")
+ }
+
/// The root directory of the application
pub(crate) fn root(&self) -> &str {
&self.root
diff --git a/frontend/rust-lib/flowy-user/src/user_manager/manager.rs b/frontend/rust-lib/flowy-user/src/user_manager/manager.rs
index 5f33a097ab..a53fc17c5d 100644
--- a/frontend/rust-lib/flowy-user/src/user_manager/manager.rs
+++ b/frontend/rust-lib/flowy-user/src/user_manager/manager.rs
@@ -25,7 +25,7 @@ use tracing::{debug, error, event, info, instrument, warn};
use uuid::Uuid;
use crate::entities::{AuthStateChangedPB, AuthStatePB, UserProfilePB, UserSettingPB};
-use crate::event_map::{DefaultUserStatusCallback, UserStatusCallback};
+use crate::event_map::{AppLifeCycle, DefaultUserStatusCallback};
use crate::migrations::document_empty_content::HistoricalEmptyDocumentMigration;
use crate::migrations::migration::{
save_migration_record, UserDataMigration, UserLocalDataMigration, FIRST_TIME_INSTALL_VERSION,
@@ -46,7 +46,7 @@ pub struct UserManager {
pub(crate) cloud_service: Weak<dyn UserCloudServiceProvider>,
pub(crate) store_preferences: Arc<KVStorePreferences>,
pub(crate) user_awareness_by_workspace: DashMap<Uuid, Arc<RwLock<UserAwareness>>>,
- pub(crate) user_status_callback: RwLock<Arc<dyn UserStatusCallback>>,
+ pub app_life_cycle: RwLock<Arc<dyn AppLifeCycle>>,
pub(crate) collab_builder: Weak<AppFlowyCollabBuilder>,
pub(crate) collab_interact: RwLock<Arc<dyn UserReminder>>,
pub(crate) user_workspace_service: Arc<dyn UserWorkspaceService>,
@@ -69,7 +69,7 @@ impl UserManager {
authenticate_user: Arc<AuthenticateUser>,
user_workspace_service: Arc<dyn UserWorkspaceService>,
) -> Arc<Self> {
- let user_status_callback: RwLock<Arc<dyn UserStatusCallback>> =
+ let user_status_callback: RwLock<Arc<dyn AppLifeCycle>> =
RwLock::new(Arc::new(DefaultUserStatusCallback));
let refresh_user_profile_since = AtomicI64::new(0);
@@ -77,7 +77,7 @@ impl UserManager {
cloud_service: cloud_services,
store_preferences,
user_awareness_by_workspace: Default::default(),
- user_status_callback,
+ app_life_cycle: user_status_callback,
collab_builder,
collab_interact: RwLock::new(Arc::new(DefaultCollabInteract)),
authenticate_user,
@@ -134,13 +134,13 @@ impl UserManager {
/// the function will set up the collaboration configuration and initialize the user's awareness. Upon successful
/// completion, a user status callback is invoked to signify that the initialization process is complete.
#[instrument(level = "debug", skip_all, err)]
- pub async fn init_with_callback<C: UserStatusCallback + 'static, I: UserReminder>(
+ pub async fn init_with_callback<C: AppLifeCycle + 'static, I: UserReminder>(
&self,
user_status_callback: C,
collab_interact: I,
) -> Result<(), FlowyError> {
- let user_status_callback = Arc::new(user_status_callback);
- *self.user_status_callback.write().await = user_status_callback.clone();
+ let app_life_cycle = Arc::new(user_status_callback);
+ *self.app_life_cycle.write().await = app_life_cycle.clone();
*self.collab_interact.write().await = Arc::new(collab_interact);
let cloud_service = self.cloud_service()?;
@@ -300,12 +300,13 @@ impl UserManager {
)
.await;
- user_status_callback
+ app_life_cycle
.on_launch_if_authenticated(
uid,
&cloud_config,
&workspace_uuid,
- &self.authenticate_user.user_config.device_id,
+ &self.authenticate_user.user_config,
+ &self.authenticate_user.user_paths,
&workspace_type,
)
.await?;
@@ -393,13 +394,14 @@ impl UserManager {
)
.await;
self
- .user_status_callback
+ .app_life_cycle
.read()
.await
.on_sign_in(
user_profile.uid,
&workspace_id,
- &self.authenticate_user.user_config.device_id,
+ &self.authenticate_user.user_config,
+ &self.authenticate_user.user_paths,
&user_profile.workspace_type,
)
.await?;
@@ -458,14 +460,15 @@ impl UserManager {
.await;
let workspace_id = Uuid::parse_str(&new_session.workspace_id)?;
self
- .user_status_callback
+ .app_life_cycle
.read()
.await
.on_sign_up(
response.is_new_user,
new_user_profile,
&workspace_id,
- &self.authenticate_user.user_config.device_id,
+ &self.authenticate_user.user_config,
+ &self.authenticate_user.user_paths,
&new_user_profile.workspace_type,
)
.await?;
diff --git a/frontend/rust-lib/flowy-user/src/user_manager/manager_user_workspace.rs b/frontend/rust-lib/flowy-user/src/user_manager/manager_user_workspace.rs
index e69dab9f76..6f61324905 100644
--- a/frontend/rust-lib/flowy-user/src/user_manager/manager_user_workspace.rs
+++ b/frontend/rust-lib/flowy-user/src/user_manager/manager_user_workspace.rs
@@ -144,6 +144,24 @@ impl UserManager {
workspace_id: &Uuid,
workspace_type: WorkspaceType,
) -> FlowyResult<()> {
+ let current_workspace_id = self.workspace_id()?;
+ if current_workspace_id != *workspace_id {
+ match self
+ .app_life_cycle
+ .read()
+ .await
+ .on_workspace_closed(&current_workspace_id)
+ .await
+ {
+ Ok(_) => {
+ trace!("close workspace: {:?}", current_workspace_id);
+ },
+ Err(err) => {
+ error!("close workspace failed: {:?}", err);
+ },
+ }
+ }
+
info!(
"open workspace: {}, auth type:{:?}",
workspace_id, workspace_type
@@ -196,10 +214,17 @@ impl UserManager {
let user_uuid = self.user_uuid()?;
if let Err(err) = self
- .user_status_callback
+ .app_life_cycle
.read()
.await
- .on_workspace_opened(uid, workspace_id, &user_workspace, &workspace_type)
+ .on_workspace_opened(
+ uid,
+ workspace_id,
+ &user_workspace,
+ &workspace_type,
+ &self.authenticate_user.user_config,
+ &self.authenticate_user.user_paths,
+ )
.await
{
error!("Open workspace failed: {:?}", err);
@@ -291,10 +316,13 @@ impl UserManager {
let conn = self.db_connection(uid)?;
delete_user_workspace(conn, workspace_id.to_string().as_str())?;
- self
- .user_workspace_service
- .did_delete_workspace(workspace_id)
+ let _ = self
+ .app_life_cycle
+ .read()
.await
+ .on_workspace_deleted(uid, workspace_id)
+ .await;
+ Ok(())
}
#[instrument(level = "info", skip(self), err)]
@@ -310,11 +338,12 @@ impl UserManager {
.delete_workspace(workspace_id)
.await?;
- self
- .user_workspace_service
- .did_delete_workspace(workspace_id)
- .await?;
-
+ let _ = self
+ .app_life_cycle
+ .read()
+ .await
+ .on_workspace_deleted(uid, workspace_id)
+ .await;
Ok(())
}
@@ -570,7 +599,7 @@ impl UserManager {
workspace_usage.storage_bytes < workspace_usage.storage_bytes_limit
};
self
- .user_status_callback
+ .app_life_cycle
.read()
.await
.on_storage_permission_updated(can_write);
@@ -736,7 +765,7 @@ impl UserManager {
trace!("Current plans: {:?}", plans);
self
- .user_status_callback
+ .app_life_cycle
.read()
.await
.on_subscription_plans_updated(plans);
diff --git a/frontend/rust-lib/lib-dispatch/src/runtime.rs b/frontend/rust-lib/lib-dispatch/src/runtime.rs
index e2f5cd56c3..648e006b63 100644
--- a/frontend/rust-lib/lib-dispatch/src/runtime.rs
+++ b/frontend/rust-lib/lib-dispatch/src/runtime.rs
@@ -7,7 +7,7 @@ use tokio::runtime::Runtime;
use tokio::task::JoinHandle;
pub struct AFPluginRuntime {
- pub(crate) inner: Runtime,
+ pub inner: Runtime,
}
impl Display for AFPluginRuntime {
diff --git a/frontend/rust-lib/rust-toolchain.toml b/frontend/rust-lib/rust-toolchain.toml
index 1de01fa45c..efd9dc3dbe 100644
--- a/frontend/rust-lib/rust-toolchain.toml
+++ b/frontend/rust-lib/rust-toolchain.toml
@@ -1,2 +1,2 @@
[toolchain]
-channel = "1.81.0"
+channel = "1.84.0"
diff --git a/frontend/rust-toolchain.toml b/frontend/rust-toolchain.toml
new file mode 100644
index 0000000000..efd9dc3dbe
--- /dev/null
+++ b/frontend/rust-toolchain.toml
@@ -0,0 +1,2 @@
+[toolchain]
+channel = "1.84.0"