Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Implement paginated query for conversation
Hakan Shehu55142f978675
diff --git a/desktop/src/components/channels/channel-container-node.tsx b/desktop/src/components/channels/channel-container-node.tsxindex 6eebe813..06939575 100644--- a/desktop/src/components/channels/channel-container-node.tsx+++ b/desktop/src/components/channels/channel-container-node.tsx@@ -7,5 +7,5 @@ interface ChannelContainerNodeProps {}export const ChannelContainerNode = ({ node }: ChannelContainerNodeProps) => {- return <Conversation nodeId={node.id} />;+ return <Conversation conversationId={node.id} />;};diff --git a/desktop/src/components/documents/document.tsx b/desktop/src/components/documents/document.tsxindex 9d86f8fe..f64d2718 100644--- a/desktop/src/components/documents/document.tsx+++ b/desktop/src/components/documents/document.tsx@@ -15,7 +15,7 @@ interface DocumentProps {export const Document = ({ node }: DocumentProps) => {const workspace = useWorkspace();const { data, isPending } = useQuery({- queryKey: [`document:nodes:${node.id}`],+ queryKey: ['document', 'nodes', node.id],queryFn: async ({ queryKey }) => {const query = sql<SelectNode>`WITH RECURSIVE document_hierarchy AS (@@ -34,8 +34,10 @@ export const Document = ({ node }: DocumentProps) => {FROM document_hierarchy;`.compile(workspace.schema);- const queryId = queryKey[0];- return await workspace.queryAndSubscribe(queryId, query);+ return await workspace.queryAndSubscribe({+ query,+ key: queryKey,+ });},});diff --git a/desktop/src/components/messages/conversation.tsx b/desktop/src/components/messages/conversation.tsxindex c00e33a2..39077d5c 100644--- a/desktop/src/components/messages/conversation.tsx+++ b/desktop/src/components/messages/conversation.tsx@@ -1,23 +1,89 @@import React from 'react';import { MessageList } from '@/components/messages/message-list';import { MessageCreate } from '@/components/messages/message-create';-import { useConversation } from '@/hooks/use-conversation';+import { ScrollArea } from '@/components/ui/scroll-area';+import { InView } from 'react-intersection-observer';interface ConversationProps {- nodeId: string;+ conversationId: string;}-export const Conversation = ({ nodeId }: ConversationProps) => {- const { messages, isLoading, createMessage } = useConversation(nodeId);+export const Conversation = ({ conversationId }: ConversationProps) => {+ const viewportRef = React.useRef<HTMLDivElement>(null);+ const containerRef = React.useRef<HTMLDivElement>(null);+ const observerRef = React.useRef<ResizeObserver | null>(null);+ const scrollPosition = React.useRef<number>(0);+ const bottomRef = React.useRef<HTMLDivElement>(null);+ const bottomVisibleRef = React.useRef<boolean>(false);+ const shouldScrollToBottom = React.useRef<boolean>(true);- if (isLoading) {- return <div>Loading...</div>;- }+ React.useEffect(() => {+ if (bottomRef.current && scrollPosition.current == 0) {+ bottomRef.current.scrollIntoView();+ }++ if (containerRef.current && viewportRef.current) {+ // observe resize of container when new messages are appended or internal elements are loaded (e.g. images)+ observerRef.current = new ResizeObserver(() => {+ if (!viewportRef.current) {+ return;+ }++ if (shouldScrollToBottom.current) {+ bottomRef.current?.scrollIntoView();+ } else {+ viewportRef.current.scrollTop =+ viewportRef.current.scrollHeight - scrollPosition.current;+ }+ });++ observerRef.current.observe(containerRef.current);++ return () => {+ if (observerRef.current) {+ observerRef.current.disconnect();+ }+ };+ }+ }, [conversationId]);++ const handleScroll = () => {+ if (viewportRef.current) {+ scrollPosition.current =+ viewportRef.current.scrollHeight - viewportRef.current.scrollTop;++ shouldScrollToBottom.current = bottomVisibleRef.current;+ }+ };return (<React.Fragment>- <MessageList conversationId={nodeId} messages={messages} />- <MessageCreate nodeId={nodeId} onSubmit={createMessage} />+ <ScrollArea+ ref={viewportRef}+ onScroll={handleScroll}+ className="flex-grow overflow-y-auto px-10"+ >+ <div className="container" ref={containerRef}>+ <MessageList+ conversationId={conversationId}+ onLastMessageIdChange={() => {+ if (shouldScrollToBottom.current && bottomRef.current) {+ bottomRef.current.scrollIntoView();+ }+ }}+ />+ </div>+ <InView+ className="h-4"+ rootMargin="20px"+ onChange={(inView) => {+ bottomVisibleRef.current = inView;+ }}+ >+ <div ref={bottomRef} className="h-4"></div>+ </InView>+ </ScrollArea>+ <MessageCreate conversationId={conversationId} /></React.Fragment>);};diff --git a/desktop/src/components/messages/message-create.tsx b/desktop/src/components/messages/message-create.tsxindex 6b9c51ef..df4209d0 100644--- a/desktop/src/components/messages/message-create.tsx+++ b/desktop/src/components/messages/message-create.tsx@@ -1,19 +1,45 @@import React from 'react';import { MessageEditor } from '@/components/messages/message-editor';import { JSONContent } from '@tiptap/core';+import { useMutation } from '@tanstack/react-query';+import { useWorkspace } from '@/contexts/workspace';+import { buildMessageCreateNodes } from '@/lib/messages';+import { CreateNode } from '@/data/schemas/workspace';interface MessageCreateProps {- nodeId: string;- onSubmit: (content: JSONContent) => void;+ conversationId: string;}-export const MessageCreate = ({ nodeId, onSubmit }: MessageCreateProps) => {+export const MessageCreate = ({ conversationId }: MessageCreateProps) => {+ const workspace = useWorkspace();+ const { mutate, isPending } = useMutation({+ mutationFn: async (content: JSONContent) => {+ const nodesToCreate: CreateNode[] = [];++ buildMessageCreateNodes(+ nodesToCreate,+ workspace.userId,+ workspace.id,+ conversationId,+ content,+ );++ const query = workspace.schema+ .insertInto('nodes')+ .values(nodesToCreate)+ .compile();++ await workspace.mutate(query);+ },+ });+return (<div className="container mt-1 px-10"><div className="flex flex-col"><MessageEditor- nodeId={nodeId}- onSubmit={onSubmit}+ conversationId={conversationId}+ onSubmit={mutate}+ loading={isPending}canEdit={true}canSubmit={true}/>diff --git a/desktop/src/components/messages/message-editor.tsx b/desktop/src/components/messages/message-editor.tsxindex f6c31086..aa27f4a7 100644--- a/desktop/src/components/messages/message-editor.tsx+++ b/desktop/src/components/messages/message-editor.tsx@@ -35,7 +35,7 @@ import {import { EditorBubbleMenu } from '@/editor/menu/bubble-menu';interface MessageEditorProps {- nodeId: string;+ conversationId: string;canSubmit?: boolean;canEdit?: boolean;loading?: boolean;@@ -83,7 +83,7 @@ export const MessageEditor = (props: MessageEditorProps) => {autofocus: 'end',editable: props.canEdit,},- [props.nodeId],+ [props.conversationId],);const handleSubmit = React.useCallback(() => {diff --git a/desktop/src/components/messages/message-list.tsx b/desktop/src/components/messages/message-list.tsxindex ad97df4e..3f603b1c 100644--- a/desktop/src/components/messages/message-list.tsx+++ b/desktop/src/components/messages/message-list.tsx@@ -1,117 +1,145 @@import React from 'react';-import { ScrollArea } from '@/components/ui/scroll-area';import { InView } from 'react-intersection-observer';import { Message } from '@/components/messages/message';-import { MessageNode } from '@/types/messages';+import { useInfiniteQuery, useQuery } from '@tanstack/react-query';+import { SelectNode } from '@/data/schemas/workspace';+import { QueryResult, sql } from 'kysely';+import { NodeTypes } from '@/lib/constants';+import { useWorkspace } from '@/contexts/workspace';+import { mapNode } from '@/lib/nodes';+import { buildMessages } from '@/lib/messages';++const MESSAGES_PER_PAGE = 50;interface MessageListProps {conversationId: string;- messages: MessageNode[];+ onLastMessageIdChange: (id: string) => void;}-export const MessageList = ({ conversationId, messages }: MessageListProps) => {- const viewportRef = React.useRef<HTMLDivElement>(null);- const containerRef = React.useRef<HTMLDivElement>(null);- const observerRef = React.useRef<ResizeObserver | null>(null);- const scrollPosition = React.useRef<number>(0);- const bottomRef = React.useRef<HTMLDivElement>(null);- const bottomVisibleRef = React.useRef<boolean>(false);- const shouldScrollToBottom = React.useRef<boolean>(true);+export const MessageList = ({+ conversationId,+ onLastMessageIdChange,+}: MessageListProps) => {+ const workspace = useWorkspace();+ const lastMessageId = React.useRef<string | null>(null);- React.useEffect(() => {- if (bottomRef.current && scrollPosition.current == 0) {- bottomRef.current.scrollIntoView();- }+ const { data, isPending, isFetchingNextPage, fetchNextPage, hasNextPage } =+ useInfiniteQuery({+ queryKey: ['conversation', conversationId],+ initialPageParam: 0,+ getNextPageParam: (lastPage: QueryResult<SelectNode>, pages) => {+ if (lastPage && lastPage.rows) {+ const messageCount = lastPage.rows.filter(+ (row) => row.type === NodeTypes.Message,+ ).length;- if (containerRef.current && viewportRef.current) {- // observe resize of container when new messages are appended or internal elements are loaded (e.g. images)- observerRef.current = new ResizeObserver(() => {- if (!viewportRef.current) {- return;- }+ if (messageCount >= MESSAGES_PER_PAGE) {+ return pages.length;+ }- if (shouldScrollToBottom) {- bottomRef.current?.scrollIntoView();- } else {- viewportRef.current.scrollTop =- viewportRef.current.scrollHeight - scrollPosition.current;+ return undefined;}- });+ return undefined;+ },+ queryFn: async ({ queryKey, pageParam }) => {+ const offset = pageParam * MESSAGES_PER_PAGE;+ const query = sql<SelectNode>`+ WITH message_nodes AS (+ SELECT *+ FROM nodes+ WHERE parent_id = ${conversationId} AND type = ${NodeTypes.Message}+ ORDER BY id DESC+ LIMIT ${sql.lit(MESSAGES_PER_PAGE)}+ OFFSET ${sql.lit(offset)}+ ),+ descendant_nodes AS (+ SELECT *+ FROM nodes+ WHERE parent_id IN (SELECT id FROM message_nodes)+ UNION ALL+ SELECT child.*+ FROM nodes child+ INNER JOIN descendant_nodes parent ON child.parent_id = parent.id+ ),+ authors AS (+ SELECT *+ FROM nodes+ WHERE id IN (SELECT DISTINCT created_by FROM message_nodes)+ )+ SELECT * FROM message_nodes+ UNION ALL+ SELECT * FROM descendant_nodes+ UNION ALL+ SELECT * FROM authors;- observerRef.current.observe(containerRef.current);+ `.compile(workspace.schema);- return () => {- if (observerRef.current) {- observerRef.current.disconnect();- }- };- }- }, [conversationId]);+ return await workspace.queryAndSubscribe({+ key: queryKey,+ page: pageParam,+ query,+ });+ },+ });- const handleScroll = () => {- if (viewportRef.current) {- scrollPosition.current =- viewportRef.current.scrollHeight - viewportRef.current.scrollTop;+ const pages = data?.pages ?? [];+ const allNodes =+ pages+ .map((page) => page.rows)+ .flat()+ .map((row) => mapNode(row)) ?? [];+ const messages = buildMessages(allNodes);- shouldScrollToBottom.current = bottomVisibleRef.current;+ React.useEffect(() => {+ if (messages.length > 0) {+ const lastMessage = messages[messages.length - 1];+ if (lastMessage.id !== lastMessageId.current) {+ lastMessageId.current = lastMessage.id;+ onLastMessageIdChange(lastMessageId.current);+ }}- };+ }, [messages]);++ if (isPending) {+ return null;+ }return (- <ScrollArea- ref={viewportRef}- onScroll={handleScroll}- className="flex-grow overflow-y-auto px-10"- >+ <React.Fragment><InViewrootMargin="200px"onChange={(inView) => {- // if (inView && hasPrevious && !isLoadingPrevious) {- // loadPrevious(50);- // }+ if (inView && hasNextPage && !isFetchingNextPage) {+ fetchNextPage();+ }}}- >- {/*<div className="flex w-full items-center justify-center py-4">*/}- {/* {isLoadingPrevious && <Loader />}*/}- {/*</div>*/}- </InView>- <div className="container" ref={containerRef}>- {messages.map((message, index) => {- const previousMessage = index > 0 ? messages[index - 1] : null;+ ></InView>+ {messages.map((message, index) => {+ const previousMessage = index > 0 ? messages[index - 1] : null;- const currentMessageDate = new Date(message.createdAt);- const previousMessageDate = previousMessage- ? new Date(previousMessage.createdAt)- : null;- const showDate =- !previousMessageDate ||- currentMessageDate.getDate() !== previousMessageDate.getDate();+ const currentMessageDate = new Date(message.createdAt);+ const previousMessageDate = previousMessage+ ? new Date(previousMessage.createdAt)+ : null;+ const showDate =+ !previousMessageDate ||+ currentMessageDate.getDate() !== previousMessageDate.getDate();- return (- <React.Fragment key={message.id}>- {showDate && (- <div className="relative flex items-center py-1">- <div className="flex-grow border-t border-gray-100" />- <span className="mx-4 flex-shrink text-xs text-muted-foreground">- {currentMessageDate.toDateString()}- </span>- <div className="flex-grow border-t border-gray-100" />- </div>- )}- <Message message={message} previousMessage={previousMessage} />- </React.Fragment>- );- })}- </div>- <InView- className="h-4"- rootMargin="20px"- onChange={(inView) => {- bottomVisibleRef.current = inView;- }}- >- <div ref={bottomRef} className="h-4"></div>- </InView>- </ScrollArea>+ return (+ <React.Fragment key={message.id}>+ {showDate && (+ <div className="relative flex items-center py-1">+ <div className="flex-grow border-t border-gray-100" />+ <span className="mx-4 flex-shrink text-xs text-muted-foreground">+ {currentMessageDate.toDateString()}+ </span>+ <div className="flex-grow border-t border-gray-100" />+ </div>+ )}+ <Message message={message} previousMessage={previousMessage} />+ </React.Fragment>+ );+ })}+ </React.Fragment>);};diff --git a/desktop/src/components/ui/scroll-area.tsx b/desktop/src/components/ui/scroll-area.tsxindex cf253cf1..5ef613f2 100644--- a/desktop/src/components/ui/scroll-area.tsx+++ b/desktop/src/components/ui/scroll-area.tsx@@ -1,7 +1,7 @@-import * as React from "react"-import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area"+import * as React from 'react';+import * as ScrollAreaPrimitive from '@radix-ui/react-scroll-area';-import { cn } from "@/lib/utils"+import { cn } from '@/lib/utils';const ScrollArea = React.forwardRef<React.ElementRef<typeof ScrollAreaPrimitive.Root>,@@ -9,38 +9,41 @@ const ScrollArea = React.forwardRef<>(({ className, children, ...props }, ref) => (<ScrollAreaPrimitive.Rootref={ref}- className={cn("relative overflow-hidden", className)}+ className={cn('relative overflow-hidden', className)}{...props}>- <ScrollAreaPrimitive.Viewport className="h-full w-full rounded-[inherit]">+ <ScrollAreaPrimitive.Viewport+ className="h-full w-full rounded-[inherit]"+ asChild+ >{children}</ScrollAreaPrimitive.Viewport><ScrollBar /><ScrollAreaPrimitive.Corner /></ScrollAreaPrimitive.Root>-))-ScrollArea.displayName = ScrollAreaPrimitive.Root.displayName+));+ScrollArea.displayName = ScrollAreaPrimitive.Root.displayName;const ScrollBar = React.forwardRef<React.ElementRef<typeof ScrollAreaPrimitive.ScrollAreaScrollbar>,React.ComponentPropsWithoutRef<typeof ScrollAreaPrimitive.ScrollAreaScrollbar>->(({ className, orientation = "vertical", ...props }, ref) => (+>(({ className, orientation = 'vertical', ...props }, ref) => (<ScrollAreaPrimitive.ScrollAreaScrollbarref={ref}orientation={orientation}className={cn(- "flex touch-none select-none transition-colors",- orientation === "vertical" &&- "h-full w-2.5 border-l border-l-transparent p-[1px]",- orientation === "horizontal" &&- "h-2.5 flex-col border-t border-t-transparent p-[1px]",- className+ 'flex touch-none select-none transition-colors',+ orientation === 'vertical' &&+ 'h-full w-2.5 border-l border-l-transparent p-[1px]',+ orientation === 'horizontal' &&+ 'h-2.5 flex-col border-t border-t-transparent p-[1px]',+ className,)}{...props}><ScrollAreaPrimitive.ScrollAreaThumb className="relative flex-1 rounded-full bg-border" /></ScrollAreaPrimitive.ScrollAreaScrollbar>-))-ScrollBar.displayName = ScrollAreaPrimitive.ScrollAreaScrollbar.displayName+));+ScrollBar.displayName = ScrollAreaPrimitive.ScrollAreaScrollbar.displayName;-export { ScrollArea, ScrollBar }+export { ScrollArea, ScrollBar };diff --git a/desktop/src/components/workspaces/container-header.tsx b/desktop/src/components/workspaces/container-header.tsxindex 3ce5eb1d..5443cc1e 100644--- a/desktop/src/components/workspaces/container-header.tsx+++ b/desktop/src/components/workspaces/container-header.tsx@@ -117,7 +117,7 @@ interface ContainerHeaderProps {export const ContainerHeader = ({ node }: ContainerHeaderProps) => {const workspace = useWorkspace();const { data, isPending } = useQuery({- queryKey: [`breadcrumb:${node.id}`],+ queryKey: ['breadcrumb', node.id],queryFn: async ({ queryKey }) => {const query = sql<SelectNode>`WITH RECURSIVE breadcrumb AS (@@ -132,8 +132,10 @@ export const ContainerHeader = ({ node }: ContainerHeaderProps) => {SELECT * FROM breadcrumb;`.compile(workspace.schema);- const queryId = queryKey[0];- return await workspace.queryAndSubscribe(queryId, query);+ return await workspace.queryAndSubscribe({+ key: queryKey,+ query,+ });},});diff --git a/desktop/src/components/workspaces/container.tsx b/desktop/src/components/workspaces/container.tsxindex 63f2aa74..a7e2acbd 100644--- a/desktop/src/components/workspaces/container.tsx+++ b/desktop/src/components/workspaces/container.tsx@@ -15,7 +15,7 @@ export const Container = () => {const workspace = useWorkspace();const { data, isPending } = useQuery({- queryKey: [`node:${nodeId}`],+ queryKey: ['node', nodeId],queryFn: async ({ queryKey }) => {const query = workspace.schema.selectFrom('nodes')@@ -23,8 +23,10 @@ export const Container = () => {.where('id', '=', nodeId).compile();- const queryId = queryKey[0];- return await workspace.queryAndSubscribe(queryId, query);+ return await workspace.queryAndSubscribe({+ key: queryKey,+ query,+ });},});diff --git a/desktop/src/components/workspaces/sidebar.tsx b/desktop/src/components/workspaces/sidebar.tsxindex d9b81e50..155a8e48 100644--- a/desktop/src/components/workspaces/sidebar.tsx+++ b/desktop/src/components/workspaces/sidebar.tsx@@ -13,7 +13,7 @@ import { SpacebarChats } from '@/components/workspaces/spacebar-chats';export const Sidebar = () => {const workspace = useWorkspace();const { data, isPending } = useQuery({- queryKey: [`sidebar:${workspace.id}`],+ queryKey: ['sidebar', workspace.id],queryFn: async ({ queryKey }) => {const query = workspace.schema.selectFrom('nodes')@@ -30,8 +30,10 @@ export const Sidebar = () => {).compile();- const queryId = queryKey[0];- return await workspace.queryAndSubscribe(queryId, query);+ return await workspace.queryAndSubscribe({+ key: queryKey,+ query,+ });},});diff --git a/desktop/src/components/workspaces/workspace.tsx b/desktop/src/components/workspaces/workspace.tsxindex fe3e2e4e..d6eb0862 100644--- a/desktop/src/components/workspaces/workspace.tsx+++ b/desktop/src/components/workspaces/workspace.tsx@@ -48,10 +48,47 @@ export const Workspace = () => {const id = eventBus.subscribe((event) => {if (event.event === 'workspace_query_updated') {const result = event.payload.result;- const queryKey = event.payload.queryId;+ const queryKey = event.payload.key;+ const page = event.payload.page;- if (result && queryKey) {- queryClient.setQueryData([queryKey], result);+ if (!result) {+ return;+ }++ if (!queryKey) {+ return;+ }++ const existingData = queryClient.getQueryData<any>(queryKey);++ if (!existingData) {+ window.neuron.unsubscribeWorkspaceQuery(+ workspace.accountId,+ workspace.id,+ queryKey,+ );+ return;+ }++ if (page !== undefined && page != null) {+ const index = existingData.pageParams.indexOf(page);+ if (index === -1) {+ return;+ }++ const newData = {+ pageParams: existingData.pageParams,+ pages: existingData.pages.map((p: any, i: number) => {+ if (i === index) {+ return result;+ }++ return p;+ }),+ };+ queryClient.setQueryData(queryKey, newData);+ } else {+ queryClient.setQueryData(queryKey, result);}}});@@ -63,11 +100,10 @@ export const Workspace = () => {event.query.queryKey &&event.query.queryKey.length > 0) {- const queryKey = event.query.queryKey[0];await window.neuron.unsubscribeWorkspaceQuery(workspace.accountId,workspace.id,- queryKey,+ event.query.queryKey,);}});@@ -99,12 +135,11 @@ export const Workspace = () => {workspace.id,query,),- queryAndSubscribe: (queryId, query) =>+ queryAndSubscribe: (context) =>window.neuron.executeWorkspaceQueryAndSubscribe(workspace.accountId,workspace.id,- queryId,- query,+ context,),navigateToNode(nodeId) {navigate(`/${workspaceId}/${nodeId}`);diff --git a/desktop/src/contexts/workspace.ts b/desktop/src/contexts/workspace.tsindex d90718fc..38c513af 100644--- a/desktop/src/contexts/workspace.ts+++ b/desktop/src/contexts/workspace.ts@@ -2,14 +2,14 @@ import { createContext, useContext } from 'react';import { Workspace } from '@/types/workspaces';import { CompiledQuery, Kysely, QueryResult } from 'kysely';import { WorkspaceDatabaseSchema } from '@/data/schemas/workspace';+import { SubscribedQueryContext } from '@/types/databases';interface WorkspaceContext extends Workspace {schema: Kysely<WorkspaceDatabaseSchema>;mutate: (mutation: CompiledQuery | CompiledQuery[]) => Promise<void>;query: <R>(query: CompiledQuery<R>) => Promise<QueryResult<R>>;queryAndSubscribe: <R>(- queryId: string,- query: CompiledQuery<R>,+ context: SubscribedQueryContext<R>,) => Promise<QueryResult<R>>;navigateToNode: (nodeId: string) => void;}diff --git a/desktop/src/data/app-manager.ts b/desktop/src/data/app-manager.tsindex d6f02c44..67322d20 100644--- a/desktop/src/data/app-manager.ts+++ b/desktop/src/data/app-manager.ts@@ -17,7 +17,7 @@ import {resultHasChanged,} from '@/data/utils';import { Workspace } from '@/types/workspaces';-import { SubscribedQueryData } from '@/types/databases';+import { SubscribedQueryResult } from '@/types/databases';import { eventBus } from '@/lib/event-bus';const EVENT_LOOP_INTERVAL = 1000;@@ -26,7 +26,7 @@ class AppManager {private readonly accounts: Map<string, AccountManager>;private readonly appPath: string;private readonly database: Kysely<AppDatabaseSchema>;- private readonly subscribers: Map<string, SubscribedQueryData<unknown>>;+ private readonly subscribers: Map<string, SubscribedQueryResult<unknown>>;private initPromise: Promise<void> | null = null;constructor() {@@ -87,8 +87,11 @@ class AppManager {}const selectedTables = extractTablesFromSql(query.sql);- const subscriberData: SubscribedQueryData<T> = {- query,+ const subscriberData: SubscribedQueryResult<T> = {+ context: {+ query,+ key: [queryId],+ },tables: selectedTables,result: result,};@@ -151,7 +154,9 @@ class AppManager {continue;}- const newResult = await this.database.executeQuery(subscriberData.query);+ const newResult = await this.database.executeQuery(+ subscriberData.context.query,+ );if (resultHasChanged(subscriberData.result, newResult)) {eventBus.publish({diff --git a/desktop/src/data/workspace-manager.ts b/desktop/src/data/workspace-manager.tsindex 69ce832a..03415c77 100644--- a/desktop/src/data/workspace-manager.ts+++ b/desktop/src/data/workspace-manager.ts@@ -15,11 +15,14 @@ import {extractTablesFromSql,resultHasChanged,} from '@/data/utils';-import { SubscribedQueryData } from '@/types/databases';+import {+ SubscribedQueryContext,+ SubscribedQueryResult,+} from '@/types/databases';import { ServerMutation } from '@/types/mutations';import { eventBus } from '@/lib/event-bus';import { AxiosInstance } from 'axios';-import { debounce } from 'lodash';+import { debounce, isEqual } from 'lodash';import { ServerNode } from '@/types/nodes';import { SelectNode } from '@/data/schemas/workspace';@@ -27,7 +30,7 @@ export class WorkspaceManager {private readonly workspace: Workspace;private readonly axios: AxiosInstance;private readonly database: Kysely<WorkspaceDatabaseSchema>;- private readonly subscribers: Map<string, SubscribedQueryData<unknown>>;+ private readonly subscribers: Map<string, SubscribedQueryResult<unknown>>;private readonly debouncedNotifyQuerySubscribers: (affectedTables: string[],) => void;@@ -78,19 +81,19 @@ export class WorkspaceManager {}public async executeQueryAndSubscribe<T>(- queryId: string,- query: CompiledQuery<T>,+ context: SubscribedQueryContext<T>,): Promise<QueryResult<T>> {- const result = await this.database.executeQuery(query);+ const result = await this.database.executeQuery(context.query);// only mutations should have side effectsif (result.numAffectedRows > 0) {throw new Error('Query should not have any side effects');}- const selectedTables = extractTablesFromSql(query.sql);- const subscriberData: SubscribedQueryData<T> = {- query,+ const queryId = context.key.join('|') + context.page;+ const selectedTables = extractTablesFromSql(context.query.sql);+ const subscriberData: SubscribedQueryResult<T> = {+ context,tables: selectedTables,result: result,};@@ -138,30 +141,39 @@ export class WorkspaceManager {}}- public unsubscribeQuery(queryId: string): void {- this.subscribers.delete(queryId);+ public unsubscribeQuery(queryKey: string[]): void {+ var queryIds = [...this.subscribers.keys()];+ for (const queryId of queryIds) {+ const subscriberData = this.subscribers.get(queryId);+ if (isEqual(subscriberData.context.key, queryKey)) {+ this.subscribers.delete(queryId);+ }+ }}private async notifyQuerySubscribers(affectedTables: string[],): Promise<void> {- for (const [subscriberId, subscriberData] of this.subscribers) {- const hasAffectedTables = subscriberData.tables.some((table) =>- affectedTables.includes(table),+ for (const subscriber of this.subscribers.values()) {+ const hasAffectedTables = affectedTables.some((table) =>+ subscriber.tables.includes(table),);if (!hasAffectedTables) {continue;}- const newResult = await this.database.executeQuery(subscriberData.query);+ const newResult = await this.database.executeQuery(+ subscriber.context.query,+ );- if (resultHasChanged(subscriberData.result, newResult)) {- subscriberData.result = newResult;+ if (resultHasChanged(subscriber.result, newResult)) {+ subscriber.result = newResult;eventBus.publish({event: 'workspace_query_updated',payload: {- queryId: subscriberId,+ key: subscriber.context.key,+ page: subscriber.context.page,result: newResult,},});diff --git a/desktop/src/hooks/use-conversation.tsx b/desktop/src/hooks/use-conversation.tsxdeleted file mode 100644index 7cb35bf6..00000000--- a/desktop/src/hooks/use-conversation.tsx+++ /dev/null@@ -1,212 +0,0 @@-import { LocalNode, NodeBlock } from '@/types/nodes';-import { useWorkspace } from '@/contexts/workspace';-import { JSONContent } from '@tiptap/core';-import { NeuronId } from '@/lib/id';-import { LeafNodeTypes, NodeTypes } from '@/lib/constants';-import {- buildNodeWithChildren,- compareNodeId,- generateNodeIndex,- mapNode,-} from '@/lib/nodes';-import { useQuery } from '@tanstack/react-query';-import { sql } from 'kysely';-import { CreateNode, SelectNode } from '@/data/schemas/workspace';-import { MessageNode } from '@/types/messages';--interface useConversationResult {- isLoading: boolean;- messages: MessageNode[];- hasMore: boolean;- loadMore: () => void;- isLoadingMore: boolean;- createMessage: (content: JSONContent) => void;-}--export const useConversation = (- conversationId: string,-): useConversationResult => {- const workspace = useWorkspace();-- const messagesQuery = useQuery({- queryKey: [`conversation:messages:${conversationId}`],- queryFn: async ({ queryKey }) => {- const query = sql<SelectNode>`- SELECT *- FROM nodes- WHERE parent_id = ${conversationId} AND type = ${NodeTypes.Message}- ORDER BY id DESC- LIMIT 50- `.compile(workspace.schema);-- const queryId = queryKey[0];- return await workspace.queryAndSubscribe(queryId, query);- },- });- const messageIds = messagesQuery.data?.rows.map((row) => row.id) ?? [];- const messageDescendantsQuery = useQuery({- queryKey: [`conversation:descendants:${messageIds.join(',')}`],- enabled: messageIds.length > 0,- queryFn: async ({ queryKey }) => {- const query = sql<SelectNode>`- WITH RECURSIVE conversation_hierarchy AS (- SELECT *- FROM nodes- WHERE parent_id IN (${sql.join(messageIds)})- UNION ALL-- SELECT child.*- FROM nodes child- INNER JOIN conversation_hierarchy parent ON child.parent_id = parent.id- )- SELECT *- FROM conversation_hierarchy;- `.compile(workspace.schema);-- const queryId = queryKey[0];- return await workspace.queryAndSubscribe(queryId, query);- },- });-- const authorIds = [- ...new Set(messagesQuery.data?.rows.map((row) => row.created_by) ?? []),- ].sort();-- const authorsQuery = useQuery({- queryKey: [`nodes:${authorIds.join(',')}`],- enabled: authorIds.length > 0,- queryFn: async ({ queryKey }) => {- const query = workspace.schema- .selectFrom('nodes')- .selectAll()- .where('id', 'in', authorIds)- .compile();-- const queryId = queryKey[0];- return await workspace.queryAndSubscribe(queryId, query);- },- });-- const createMessage = async (content: JSONContent) => {- const nodesToInsert: CreateNode[] = [];-- buildMessageCreateNodes(- nodesToInsert,- workspace.userId,- workspace.id,- conversationId,- content,- );-- const query = workspace.schema- .insertInto('nodes')- .values(nodesToInsert)- .compile();- await workspace.mutate(query);- };-- const messageNodes =- messagesQuery.data?.rows.map((row) => mapNode(row)) ?? [];- const descendentNodes =- messageDescendantsQuery.data?.rows.map((row) => mapNode(row)) ?? [];- const authorNodes = authorsQuery.data?.rows.map((row) => mapNode(row)) ?? [];- const messages = buildMessages(messageNodes, descendentNodes, authorNodes);-- return {- isLoading: messagesQuery.isPending,- messages: messages,- hasMore: false,- loadMore: () => console.log('load more'),- isLoadingMore: false,- createMessage: createMessage,- };-};--const buildMessages = (- messageNodes: LocalNode[],- descendentNodes: LocalNode[],- authorNodes: LocalNode[],-): MessageNode[] => {- const messages: MessageNode[] = [];- const authorMap = new Map<string, LocalNode>();-- for (const author of authorNodes) {- authorMap.set(author.id, author);- }-- for (const node of messageNodes) {- const messageNode = buildNodeWithChildren(node, descendentNodes);- const author = authorMap.get(node.createdBy);- const message: MessageNode = {- ...messageNode,- author,- };-- messages.push(message);- }-- return messages.sort((a, b) => compareNodeId(a, b));-};--const buildMessageCreateNodes = (- nodes: CreateNode[],- userId: string,- workspaceId: string,- parentId: string,- content: JSONContent,- index?: string | null,-) => {- const id =- content.attrs?.id ??- NeuronId.generate(NeuronId.getIdTypeFromNode(content.type));-- let attrs = content.attrs ? { ...content.attrs } : null;- if (attrs) {- delete attrs.id;-- if (Object.keys(attrs).length === 0) {- attrs = null;- }- }-- let nodeContent: NodeBlock[] | null = null;- if (LeafNodeTypes.includes(content.type)) {- nodeContent = [];- for (const child of content.content) {- nodeContent.push({- type: child.type,- text: child.text,- marks: child.marks?.map((mark) => {- return {- type: mark.type,- attrs: mark.attrs,- };- }),- });- }- }-- if (nodeContent && nodeContent.length === 0) {- nodeContent = null;- }-- nodes.push({- id: id,- parent_id: parentId,- type: content.type,- attrs: attrs ? JSON.stringify(attrs) : null,- index: index,- content: nodeContent ? JSON.stringify(nodeContent) : null,- created_at: new Date().toISOString(),- created_by: userId,- version_id: NeuronId.generate(NeuronId.Type.Version),- });-- if (nodeContent == null && content.content && content.content.length > 0) {- let lastIndex: string | null = null;- for (const child of content.content) {- lastIndex = generateNodeIndex(lastIndex, null);- buildMessageCreateNodes(nodes, userId, workspaceId, id, child, lastIndex);- }- }-};diff --git a/desktop/src/lib/messages.ts b/desktop/src/lib/messages.tsnew file mode 100644index 00000000..a6d29e58--- /dev/null+++ b/desktop/src/lib/messages.ts@@ -0,0 +1,101 @@+import { CreateNode } from '@/data/schemas/workspace';+import { JSONContent } from '@tiptap/core';+import { NeuronId } from '@/lib/id';+import { LocalNode, NodeBlock } from '@/types/nodes';+import { LeafNodeTypes, NodeTypes } from '@/lib/constants';+import {+ buildNodeWithChildren,+ compareNodeId,+ generateNodeIndex,+} from '@/lib/nodes';+import { MessageNode } from '@/types/messages';++export const buildMessageCreateNodes = (+ nodes: CreateNode[],+ userId: string,+ workspaceId: string,+ parentId: string,+ content: JSONContent,+ index?: string | null,+) => {+ const id =+ content.attrs?.id ??+ NeuronId.generate(NeuronId.getIdTypeFromNode(content.type));++ let attrs = content.attrs ? { ...content.attrs } : null;+ if (attrs) {+ delete attrs.id;++ if (Object.keys(attrs).length === 0) {+ attrs = null;+ }+ }++ let nodeContent: NodeBlock[] | null = null;+ if (LeafNodeTypes.includes(content.type)) {+ nodeContent = [];+ for (const child of content.content) {+ nodeContent.push({+ type: child.type,+ text: child.text,+ marks: child.marks?.map((mark) => {+ return {+ type: mark.type,+ attrs: mark.attrs,+ };+ }),+ });+ }+ }++ if (nodeContent && nodeContent.length === 0) {+ nodeContent = null;+ }++ nodes.push({+ id: id,+ parent_id: parentId,+ type: content.type,+ attrs: attrs ? JSON.stringify(attrs) : null,+ index: index,+ content: nodeContent ? JSON.stringify(nodeContent) : null,+ created_at: new Date().toISOString(),+ created_by: userId,+ version_id: NeuronId.generate(NeuronId.Type.Version),+ });++ if (nodeContent == null && content.content && content.content.length > 0) {+ let lastIndex: string | null = null;+ for (const child of content.content) {+ lastIndex = generateNodeIndex(lastIndex, null);+ buildMessageCreateNodes(nodes, userId, workspaceId, id, child, lastIndex);+ }+ }+};++export const buildMessages = (allNodes: LocalNode[]): MessageNode[] => {+ const messageNodes = allNodes+ .filter((node) => node.type === NodeTypes.Message)+ .sort((a, b) => compareNodeId(a, b));++ const authorNodes = allNodes.filter((node) => node.type === NodeTypes.User);+ const messages: MessageNode[] = [];+ const authorMap = new Map<string, LocalNode>();++ for (const author of authorNodes) {+ authorMap.set(author.id, author);+ }++ for (const node of messageNodes) {+ const messageNode = buildNodeWithChildren(node, allNodes);+ const author = authorMap.get(node.createdBy);+ const message: MessageNode = {+ ...messageNode,+ author,+ };++ messages.push(message);+ }++ return messages.sort((a, b) => compareNodeId(a, b));+};diff --git a/desktop/src/main.ts b/desktop/src/main.tsindex 1900acab..4b58782e 100644--- a/desktop/src/main.ts+++ b/desktop/src/main.ts@@ -3,6 +3,7 @@ import path from 'path';import { eventBus } from '@/lib/event-bus';import { appManager } from '@/data/app-manager';import { CompiledQuery, QueryResult } from 'kysely';+import { SubscribedQueryContext } from './types/databases';let subscriptionId: string | null = null;@@ -148,8 +149,7 @@ ipcMain.handle(_,accountId: string,workspaceId: string,- queryId: string,- query: CompiledQuery<unknown>,+ context: SubscribedQueryContext<unknown>,): Promise<QueryResult<unknown>> => {const accountManager = await appManager.getAccount(accountId);if (!accountManager) {@@ -161,7 +161,7 @@ ipcMain.handle(throw new Error(`Workspace not found: ${workspaceId}`);}- return await workspaceManager.executeQueryAndSubscribe(queryId, query);+ return await workspaceManager.executeQueryAndSubscribe(context);},);@@ -171,7 +171,7 @@ ipcMain.handle(_,accountId: string,workspaceId: string,- queryId: string,+ queryKey: string[],): Promise<void> => {const accountManager = await appManager.getAccount(accountId);if (!accountManager) {@@ -183,6 +183,6 @@ ipcMain.handle(throw new Error(`Workspace not found: ${workspaceId}`);}- workspaceManager.unsubscribeQuery(queryId);+ workspaceManager.unsubscribeQuery(queryKey);},);diff --git a/desktop/src/types/databases.ts b/desktop/src/types/databases.tsindex b7c91e60..b5a13c0a 100644--- a/desktop/src/types/databases.ts+++ b/desktop/src/types/databases.ts@@ -1,7 +1,13 @@import { CompiledQuery, QueryResult } from 'kysely';-export type SubscribedQueryData<T> = {+export type SubscribedQueryContext<T> = {+ key: string[];+ page?: number;query: CompiledQuery<T>;+};++export type SubscribedQueryResult<T> = {+ context: SubscribedQueryContext<T>;tables: string[];result: QueryResult<T>;};diff --git a/desktop/src/types/window.d.ts b/desktop/src/types/window.d.tsindex 07d5f836..482bcd14 100644--- a/desktop/src/types/window.d.ts+++ b/desktop/src/types/window.d.ts@@ -1,6 +1,7 @@import { EventBus } from '@/lib/event-bus';import { CompiledQuery, QueryResult } from 'kysely';import { LocalMutationInput } from '@/types/mutations';+import { SubscribedQueryContext } from './databases';interface NeuronApi {init: () => Promise<void>;@@ -28,14 +29,13 @@ interface NeuronApi {executeWorkspaceQueryAndSubscribe: (accountId: string,workspaceId: string,- queryId: string,- query: CompiledQuery<R>,+ context: SubscribedQueryContext<R>,) => Promise<QueryResult<R>>;unsubscribeWorkspaceQuery: (accountId: string,workspaceId: string,- queryId: string,+ queryKey: string[],) => Promise<void>;}