Skip to content

Bring projects, wikis, and teams together with AI. AppFlowy is the AI collaborative workspace where you achieve more without losing control of your data. The leading open source Notion alternative.

chore: show attachment on local ai (#5929)

Nathan.fooof7adcae8ffd4
diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_ai_message_bloc.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_ai_message_bloc.dart
index 9083e40521..19fff60bbf 100644
--- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_ai_message_bloc.dart
+++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_ai_message_bloc.dart
@@ -16,13 +16,13 @@ part 'chat_ai_message_bloc.freezed.dart';
class ChatAIMessageBloc extends Bloc<ChatAIMessageEvent, ChatAIMessageState> {
ChatAIMessageBloc({
dynamic message,
- String? metadata,
+ String? refSourceJsonString,
required this.chatId,
required this.questionId,
}) : super(
ChatAIMessageState.initial(
message,
- messageRefSourceFromString(metadata),
+ messageReferenceSource(refSourceJsonString),
),
) {
if (state.stream != null) {
diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart
index cd5579d7d4..09bf21acaa 100644
--- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart
+++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_bloc.dart
@@ -374,7 +374,10 @@ class ChatBloc extends Bloc<ChatEvent, ChatState> {
metadata: await metadataPBFromMetadata(metadata),
);
- final questionStreamMessage = _createQuestionStreamMessage(questionStream);
+ final questionStreamMessage = _createQuestionStreamMessage(
+ questionStream,
+ metadata,
+ );
add(ChatEvent.receveMessage(questionStreamMessage));
// Stream message to the server
@@ -432,14 +435,24 @@ class ChatBloc extends Bloc<ChatEvent, ChatState> {
);
}
- Message _createQuestionStreamMessage(QuestionStream stream) {
+ Message _createQuestionStreamMessage(
+ QuestionStream stream,
+ Map<String, dynamic>? sentMetadata,
+ ) {
questionStreamMessageId = nanoid();
+ final Map<String, dynamic> metadata = {};
+
+ // if (sentMetadata != null) {
+ // metadata[messageMetadataJsonStringKey] = sentMetadata;
+ // }
+
+ metadata["$QuestionStream"] = stream;
+ metadata["chatId"] = chatId;
+ metadata[messageChatFileListKey] =
+ chatFilesFromMessageMetadata(sentMetadata);
return TextMessage(
author: User(id: state.userProfile.id.toString()),
- metadata: {
- "$QuestionStream": stream,
- "chatId": chatId,
- },
+ metadata: metadata,
id: questionStreamMessageId,
createdAt: DateTime.now().millisecondsSinceEpoch,
text: '',
@@ -460,7 +473,7 @@ class ChatBloc extends Bloc<ChatEvent, ChatState> {
text: message.content,
createdAt: message.createdAt.toInt() * 1000,
metadata: {
- messageMetadataKey: message.metadata,
+ messageRefSourceJsonStringKey: message.metadata,
},
);
}
diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_entity.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_entity.dart
index 99b6b66bb1..c0a2d5bf32 100644
--- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_entity.dart
+++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_entity.dart
@@ -15,8 +15,14 @@ part 'chat_entity.freezed.dart';
const sendMessageErrorKey = "sendMessageError";
const systemUserId = "system";
const aiResponseUserId = "0";
-const messageMetadataKey = "metadata";
-const messageQuestionIdKey = "question";
+
+/// `messageRefSourceJsonStringKey` is the key used for metadata that contains the reference source of a message.
+/// Each message may include this information.
+/// - When used in a sent message, it indicates that the message includes an attachment.
+/// - When used in a received message, it indicates the AI reference sources used to answer a question.
+const messageRefSourceJsonStringKey = "ref_source_json_string";
+const messageChatFileListKey = "chat_files";
+const messageQuestionIdKey = "question_id";
@JsonSerializable()
class ChatMessageRefSource {
diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_service.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_service.dart
index a2cfc9f986..c497cd6c23 100644
--- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_service.dart
+++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_service.dart
@@ -32,7 +32,12 @@ List<ChatFile> chatFilesFromMetadataString(String? s) {
final metadataJson = jsonDecode(s);
if (metadataJson is Map<String, dynamic>) {
- return _parseChatFile(metadataJson);
+ final file = chatFileFromMap(metadataJson);
+ if (file != null) {
+ return [file];
+ } else {
+ return [];
+ }
} else if (metadataJson is List) {
return metadataJson
.map((e) => e as Map<String, dynamic>)
@@ -46,11 +51,6 @@ List<ChatFile> chatFilesFromMetadataString(String? s) {
}
}
-List<ChatFile> _parseChatFile(Map<String, dynamic> map) {
- final file = chatFileFromMap(map);
- return file != null ? [file] : [];
-}
-
ChatFile? chatFileFromMap(Map<String, dynamic>? map) {
if (map == null) return null;
@@ -63,7 +63,7 @@ ChatFile? chatFileFromMap(Map<String, dynamic>? map) {
return ChatFile.fromFilePath(filePath);
}
-List<ChatMessageRefSource> messageRefSourceFromString(String? s) {
+List<ChatMessageRefSource> messageReferenceSource(String? s) {
if (s == null || s.isEmpty || s == "null") {
return [];
}
@@ -139,3 +139,18 @@ Future<List<ChatMessageMetaPB>> metadataPBFromMetadata(
return metadata;
}
+
+List<ChatFile> chatFilesFromMessageMetadata(
+ Map<String, dynamic>? map,
+) {
+ final List<ChatFile> metadata = [];
+ if (map != null) {
+ for (final entry in map.entries) {
+ if (entry.value is ChatFile) {
+ metadata.add(entry.value);
+ }
+ }
+ }
+
+ return metadata;
+}
diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_stream.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_stream.dart
index a82a84f34a..438f3874e2 100644
--- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_stream.dart
+++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_message_stream.dart
@@ -25,7 +25,7 @@ class AnswerStream {
} else if (event.startsWith("metadata:")) {
if (_onMetadata != null) {
final s = event.substring(9);
- _onMetadata!(messageRefSourceFromString(s));
+ _onMetadata!(messageReferenceSource(s));
}
} else if (event == "AI_RESPONSE_LIMIT") {
if (_onAIResponseLimit != null) {
diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_user_message_bubble_bloc.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_user_message_bubble_bloc.dart
index 542280ca99..c4571915df 100644
--- a/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_user_message_bubble_bloc.dart
+++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/application/chat_user_message_bubble_bloc.dart
@@ -11,11 +11,10 @@ class ChatUserMessageBubbleBloc
extends Bloc<ChatUserMessageBubbleEvent, ChatUserMessageBubbleState> {
ChatUserMessageBubbleBloc({
required Message message,
- required String? metadata,
}) : super(
ChatUserMessageBubbleState.initial(
message,
- chatFilesFromMetadataString(metadata),
+ _getFiles(message.metadata),
),
) {
on<ChatUserMessageBubbleEvent>(
@@ -28,6 +27,19 @@ class ChatUserMessageBubbleBloc
}
}
+List<ChatFile> _getFiles(Map<String, dynamic>? metadata) {
+ if (metadata == null) {
+ return [];
+ }
+ final refSourceMetadata = metadata[messageRefSourceJsonStringKey] as String?;
+ final files = metadata[messageChatFileListKey] as List<ChatFile>?;
+
+ if (refSourceMetadata != null) {
+ return chatFilesFromMetadataString(refSourceMetadata);
+ }
+ return files ?? [];
+}
+
@freezed
class ChatUserMessageBubbleEvent with _$ChatUserMessageBubbleEvent {
const factory ChatUserMessageBubbleEvent.initial() = Initial;
diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart
index 6ad35d08db..fced505f20 100644
--- a/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart
+++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/chat_page.dart
@@ -320,17 +320,16 @@ class _ChatContentPageState extends State<_ChatContentPage> {
Widget _buildTextMessage(BuildContext context, TextMessage message) {
if (message.author.id == _user.id) {
final stream = message.metadata?["$QuestionStream"];
- final metadata = message.metadata?[messageMetadataKey] as String?;
return ChatUserMessageWidget(
key: ValueKey(message.id),
user: message.author,
message: stream is QuestionStream ? stream : message.text,
- metadata: metadata,
);
} else {
final stream = message.metadata?["$AnswerStream"];
final questionId = message.metadata?[messageQuestionIdKey];
- final metadata = message.metadata?[messageMetadataKey] as String?;
+ final refSourceJsonString =
+ message.metadata?[messageRefSourceJsonStringKey] as String?;
return ChatAIMessageWidget(
user: message.author,
messageUserId: message.id,
@@ -338,7 +337,7 @@ class _ChatContentPageState extends State<_ChatContentPage> {
key: ValueKey(message.id),
questionId: questionId,
chatId: widget.view.id,
- metadata: metadata,
+ refSourceJsonString: refSourceJsonString,
onSelectedMetadata: (ChatMessageRefSource metadata) {
context.read<ChatSidePannelBloc>().add(
ChatSidePannelEvent.selectedMetadata(metadata),
diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/ai_text_message.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/ai_text_message.dart
index ed928eba14..7aa48b094f 100644
--- a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/ai_text_message.dart
+++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/ai_text_message.dart
@@ -22,7 +22,7 @@ class ChatAIMessageWidget extends StatelessWidget {
required this.message,
required this.questionId,
required this.chatId,
- required this.metadata,
+ required this.refSourceJsonString,
required this.onSelectedMetadata,
});
@@ -33,7 +33,7 @@ class ChatAIMessageWidget extends StatelessWidget {
final dynamic message;
final Int64? questionId;
final String chatId;
- final String? metadata;
+ final String? refSourceJsonString;
final void Function(ChatMessageRefSource metadata) onSelectedMetadata;
@override
@@ -41,7 +41,7 @@ class ChatAIMessageWidget extends StatelessWidget {
return BlocProvider(
create: (context) => ChatAIMessageBloc(
message: message,
- metadata: metadata,
+ refSourceJsonString: refSourceJsonString,
chatId: chatId,
questionId: questionId,
)..add(const ChatAIMessageEvent.initial()),
diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_message_bubble.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_message_bubble.dart
index 06401b76a7..08c627b0b7 100644
--- a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_message_bubble.dart
+++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_message_bubble.dart
@@ -29,12 +29,10 @@ class ChatUserMessageBubble extends StatelessWidget {
.read<ChatMemberBloc>()
.add(ChatMemberEvent.getMemberInfo(message.author.id));
}
- final metadata = message.metadata?[messageMetadataKey] as String?;
return BlocProvider(
create: (context) => ChatUserMessageBubbleBloc(
message: message,
- metadata: metadata,
),
child: BlocBuilder<ChatUserMessageBubbleBloc, ChatUserMessageBubbleState>(
builder: (context, state) {
diff --git a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_text_message.dart b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_text_message.dart
index 0b2a2efa7d..c804441188 100644
--- a/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_text_message.dart
+++ b/frontend/appflowy_flutter/lib/plugins/ai_chat/presentation/message/user_text_message.dart
@@ -10,12 +10,10 @@ class ChatUserMessageWidget extends StatelessWidget {
super.key,
required this.user,
required this.message,
- required this.metadata,
});
final User user;
final dynamic message;
- final String? metadata;
@override
Widget build(BuildContext context) {
diff --git a/frontend/rust-lib/flowy-ai/src/middleware/chat_service_mw.rs b/frontend/rust-lib/flowy-ai/src/middleware/chat_service_mw.rs
index 8da2505e75..2e7f4e54a2 100644
--- a/frontend/rust-lib/flowy-ai/src/middleware/chat_service_mw.rs
+++ b/frontend/rust-lib/flowy-ai/src/middleware/chat_service_mw.rs
@@ -63,6 +63,7 @@ impl AICloudServiceMiddleware {
let _ = index_process_sink
.send(StreamMessage::IndexStart.to_string())
.await;
+
self
.local_llm_controller
.index_message_metadata(chat_id, metadata_list, index_process_sink)