Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

editor: support audio attachment playback

01zulfiadd1b8a68aa5
diff --git a/apps/web/src/components/editor/picker.ts b/apps/web/src/components/editor/picker.ts
index bd91d2a37..27b480629 100644
--- a/apps/web/src/components/editor/picker.ts
+++ b/apps/web/src/components/editor/picker.ts
@@ -87,12 +87,16 @@ export async function attachFiles(
: [];
}
- const documents = files.filter((f) => !f.type.startsWith("image/"));
+ const documents = files.filter(
+ (f) => !f.type.startsWith("image/") && !f.type.startsWith("audio/")
+ );
+ const audios = files.filter((f) => f.type.startsWith("audio/"));
const attachments: Attachment[] = [];
- for (const file of [...images, ...documents]) {
- const attachment = !skipSpecialImageHandling && file.type.startsWith("image/")
- ? await pickImage(file)
- : await pickFile(file);
+ for (const file of [...images, ...documents, ...audios]) {
+ const attachment =
+ !skipSpecialImageHandling && file.type.startsWith("image/")
+ ? await pickImage(file)
+ : await pickFile(file);
if (!attachment) continue;
attachments.push(attachment);
}
@@ -136,7 +140,7 @@ async function pickFile(
const hash = await addAttachment(file, options);
return {
- type: "file",
+ type: file.type.startsWith("audio") ? "audio" : "file",
filename: file.name,
hash,
mime: file.type,
diff --git a/apps/web/src/components/editor/tiptap.tsx b/apps/web/src/components/editor/tiptap.tsx
index d32117318..6f1457cf1 100644
--- a/apps/web/src/components/editor/tiptap.tsx
+++ b/apps/web/src/components/editor/tiptap.tsx
@@ -728,6 +728,8 @@ function toIEditor(editor: Editor): IEditor {
attachFile: (file: Attachment) =>
file.type === "image"
? editor.commands.insertImage(file)
+ : file.type === "audio"
+ ? editor.commands.insertAudio(file)
: editor.commands.insertAttachment(file),
sendAttachmentProgress: (hash, progress) =>
editor.commands.updateAttachment(
diff --git a/packages/editor/src/extensions/attachment/attachment.ts b/packages/editor/src/extensions/attachment/attachment.ts
index b17bd72de..fd14eb88e 100644
--- a/packages/editor/src/extensions/attachment/attachment.ts
+++ b/packages/editor/src/extensions/attachment/attachment.ts
@@ -85,7 +85,14 @@ export const AttachmentNode = Node.create<AttachmentOptions>({
parseHTML() {
return [
{
- tag: "span[data-hash]"
+ tag: "span[data-hash]",
+ getAttrs: (dom) => {
+ const element = dom as HTMLElement;
+ if (element.dataset.mime?.startsWith("audio/")) {
+ return false;
+ }
+ return {};
+ }
}
];
},
diff --git a/packages/editor/src/extensions/attachment/component.tsx b/packages/editor/src/extensions/attachment/component.tsx
index 2d0431bce..b542ef920 100644
--- a/packages/editor/src/extensions/attachment/component.tsx
+++ b/packages/editor/src/extensions/attachment/component.tsx
@@ -25,6 +25,7 @@ import { Icons } from "../../toolbar/icons.js";
import { ReactNodeViewProps } from "../react/index.js";
import { ToolbarGroup } from "../../toolbar/components/toolbar-group.js";
import { DesktopOnly } from "../../components/responsive/index.js";
+import { formatBytes } from "@notesnook/common";
export function AttachmentComponent(props: ReactNodeViewProps<FileAttachment>) {
const { editor, node, selected } = props;
@@ -82,7 +83,7 @@ export function AttachmentComponent(props: ReactNodeViewProps<FileAttachment>) {
flexShrink: 0
}}
>
- {progress ? `${progress}%` : formatBytes(size)}
+ {progress ? `${progress}%` : formatBytes(size, 1)}
</Text>
<DesktopOnly>
{selected && !isDragging && (
@@ -111,15 +112,3 @@ export function AttachmentComponent(props: ReactNodeViewProps<FileAttachment>) {
</Box>
);
}
-
-function formatBytes(bytes: number, decimals = 1) {
- if (bytes === 0) return "0B";
-
- const k = 1024;
- const dm = decimals < 0 ? 0 : decimals;
- const sizes = ["B", "K", "M", "G", "T", "P", "E", "Z", "Y"];
-
- const i = Math.floor(Math.log(bytes) / Math.log(k));
-
- return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + sizes[i];
-}
diff --git a/packages/editor/src/extensions/attachment/types.ts b/packages/editor/src/extensions/attachment/types.ts
index 8d8429b52..e941fb9e0 100644
--- a/packages/editor/src/extensions/attachment/types.ts
+++ b/packages/editor/src/extensions/attachment/types.ts
@@ -29,6 +29,10 @@ export type FileAttachment = BaseAttachment & {
type: "file";
};
+export type AudioAttachment = BaseAttachment & {
+ type: "audio";
+};
+
export type WebClipAttachment = BaseAttachment & {
type: "web-clip";
src: string;
@@ -50,4 +54,8 @@ export type ImageAlignmentOptions = {
align?: "center" | "left" | "right";
};
-export type Attachment = FileAttachment | WebClipAttachment | ImageAttachment;
+export type Attachment =
+ | FileAttachment
+ | AudioAttachment
+ | WebClipAttachment
+ | ImageAttachment;
diff --git a/packages/editor/src/extensions/audio/audio.ts b/packages/editor/src/extensions/audio/audio.ts
new file mode 100644
index 000000000..3dd952bf0
--- /dev/null
+++ b/packages/editor/src/extensions/audio/audio.ts
@@ -0,0 +1,105 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Node, mergeAttributes } from "@tiptap/core";
+import { hasSameAttributes } from "../../utils/prosemirror.js";
+import { AudioAttachment, getDataAttribute } from "../attachment/index.js";
+import { createNodeView } from "../react/index.js";
+import { AudioComponent } from "./component.js";
+
+export interface AudioOptions {
+ HTMLAttributes: Record<string, unknown>;
+}
+
+export type AudioAttributes = AudioAttachment;
+
+declare module "@tiptap/core" {
+ interface Commands<ReturnType> {
+ audio: {
+ insertAudio: (audio: AudioAttachment) => ReturnType;
+ };
+ }
+}
+
+export const AudioNode = Node.create<AudioOptions>({
+ name: "audio",
+ content: "",
+ marks: "",
+ draggable: true,
+ priority: 51,
+
+ addOptions() {
+ return {
+ HTMLAttributes: {}
+ };
+ },
+
+ group() {
+ return "block";
+ },
+
+ addAttributes() {
+ return {
+ type: { default: "audio", rendered: false },
+ progress: {
+ default: 0,
+ rendered: false
+ },
+ filename: getDataAttribute("filename"),
+ size: getDataAttribute("size"),
+ hash: getDataAttribute("hash"),
+ mime: getDataAttribute("mime")
+ };
+ },
+
+ parseHTML() {
+ return [
+ {
+ tag: "audio[data-hash][data-mime^='audio/']"
+ }
+ ];
+ },
+
+ renderHTML({ HTMLAttributes }) {
+ return [
+ "audio",
+ mergeAttributes(this.options.HTMLAttributes, HTMLAttributes)
+ ];
+ },
+
+ addCommands() {
+ return {
+ insertAudio:
+ (audio) =>
+ ({ commands }) => {
+ return commands.insertContent({
+ type: this.name,
+ attrs: audio
+ });
+ }
+ };
+ },
+
+ addNodeView() {
+ return createNodeView(AudioComponent, {
+ shouldUpdate: (prev, next) => !hasSameAttributes(prev.attrs, next.attrs),
+ forceEnableSelection: true
+ });
+ }
+});
diff --git a/packages/editor/src/extensions/audio/component.tsx b/packages/editor/src/extensions/audio/component.tsx
new file mode 100644
index 000000000..f0f90765e
--- /dev/null
+++ b/packages/editor/src/extensions/audio/component.tsx
@@ -0,0 +1,159 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Box, Text } from "@theme-ui/components";
+import { AudioAttachment } from "../attachment/types.js";
+import { useRef, useState, useEffect } from "react";
+import { Icon } from "@notesnook/ui";
+import { Icons } from "../../toolbar/icons.js";
+import { ReactNodeViewProps } from "../react/index.js";
+import { ToolbarGroup } from "../../toolbar/components/toolbar-group.js";
+import { DesktopOnly } from "../../components/responsive/index.js";
+import { toBlobURL, revokeBloburl } from "../../utils/downloader.js";
+import { formatBytes } from "@notesnook/common";
+
+export function AudioComponent(props: ReactNodeViewProps<AudioAttachment>) {
+ const { editor, node, selected } = props;
+ const { filename, size, progress, mime, hash } = node.attrs;
+ const elementRef = useRef<HTMLDivElement>();
+ const [isDragging, setIsDragging] = useState(false);
+ const [audioSrc, setAudioSrc] = useState<string>();
+
+ useEffect(() => {
+ if (editor.storage?.getAttachmentData && hash) {
+ editor.storage
+ .getAttachmentData({
+ type: "file",
+ hash
+ })
+ .then((data: string | undefined) => {
+ if (data) {
+ const url = toBlobURL(data, "other", mime, hash);
+ if (url) {
+ setAudioSrc(url);
+ }
+ }
+ })
+ .catch(console.error);
+ }
+ }, [editor.storage, hash, mime]);
+
+ useEffect(() => {
+ return () => {
+ if (audioSrc && hash) {
+ revokeBloburl(hash);
+ }
+ };
+ }, [audioSrc, hash]);
+
+ return (
+ <Box
+ ref={elementRef}
+ contentEditable={false}
+ draggable="false"
+ sx={{
+ display: "flex",
+ flexDirection: "column",
+ position: "relative",
+ userSelect: "none",
+ backgroundColor: "var(--background-secondary)",
+ p: 1,
+ borderRadius: "default",
+ border: "1px solid var(--border)",
+ borderColor: selected ? "accent" : "border",
+ width: "100%",
+ maxWidth: 500,
+ ":hover": {
+ backgroundColor: "var(--hover)"
+ }
+ }}
+ onDragStart={() => setIsDragging(true)}
+ onDragEnd={() => setIsDragging(false)}
+ data-drag-handle
+ >
+ <Box
+ sx={{
+ display: "flex",
+ alignItems: "center",
+ mb: audioSrc ? 2 : 0
+ }}
+ >
+ <Icon path={Icons.attachment} size={16} />
+ <Text
+ as="span"
+ sx={{
+ ml: "small",
+ fontSize: "body",
+ whiteSpace: "nowrap",
+ textOverflow: "ellipsis",
+ overflow: "hidden",
+ flex: 1
+ }}
+ >
+ {filename}
+ </Text>
+ <Text
+ as="span"
+ sx={{
+ ml: 1,
+ fontSize: "0.65rem",
+ color: "var(--paragraph-secondary)",
+ flexShrink: 0
+ }}
+ >
+ {progress ? `${progress}%` : formatBytes(size, 1)}
+ </Text>
+ </Box>
+ {audioSrc && (
+ <Box
+ sx={{
+ width: "100%",
+ "& audio": {
+ width: "100%",
+ height: 35
+ }
+ }}
+ >
+ <audio controls preload="metadata" src={audioSrc} />
+ </Box>
+ )}
+ <DesktopOnly>
+ {selected && !isDragging && (
+ <ToolbarGroup
+ editor={editor}
+ groupId="audioTools"
+ tools={
+ editor.isEditable
+ ? ["removeAttachment", "downloadAttachment"]
+ : ["downloadAttachment"]
+ }
+ sx={{
+ boxShadow: "menu",
+ borderRadius: "default",
+ backgroundColor: "var(--background)",
+ position: "absolute",
+ top: -35,
+ right: 0
+ }}
+ />
+ )}
+ </DesktopOnly>
+ </Box>
+ );
+}
diff --git a/packages/editor/src/extensions/audio/index.ts b/packages/editor/src/extensions/audio/index.ts
new file mode 100644
index 000000000..7079357e4
--- /dev/null
+++ b/packages/editor/src/extensions/audio/index.ts
@@ -0,0 +1,20 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+export * from "./audio.js";
diff --git a/packages/editor/src/index.ts b/packages/editor/src/index.ts
index b8dd1fb2c..122c4bc77 100644
--- a/packages/editor/src/index.ts
+++ b/packages/editor/src/index.ts
@@ -66,6 +66,7 @@ import { TaskItemNode } from "./extensions/task-item/index.js";
import { TaskListNode } from "./extensions/task-list/index.js";
import TextDirection from "./extensions/text-direction/index.js";
import { WebClipNode, WebClipOptions } from "./extensions/web-clip/index.js";
+import { AudioNode, AudioOptions } from "./extensions/audio/index.js";
import { useEditor } from "./hooks/use-editor.js";
import { usePermissionHandler } from "./hooks/use-permission-handler.js";
import Toolbar from "./toolbar/index.js";
@@ -125,6 +126,7 @@ const CoreExtensions = Object.entries(TiptapCoreExtensions)
export type TiptapOptions = EditorOptions &
Omit<WebClipOptions, "HTMLAttributes"> &
+ Omit<AudioOptions, "HTMLAttributes"> &
Omit<ImageOptions, "HTMLAttributes"> &
DateTimeOptions &
TiptapStorage & {
@@ -312,7 +314,12 @@ const useTiptap = (
ImageNode.configure({ allowBase64: true }),
EmbedNode,
AttachmentNode.configure({
- types: [AttachmentNode.name, ImageNode.name, WebClipNode.name]
+ types: [
+ AttachmentNode.name,
+ ImageNode.name,
+ WebClipNode.name,
+ AudioNode.name
+ ]
}),
OutlineListItem,
OutlineList.configure({ keepAttributes: true, keepMarks: true }),
@@ -327,6 +334,7 @@ const useTiptap = (
DateTime.configure({ dateFormat, timeFormat, dayFormat }),
KeyMap,
WebClipNode,
+ AudioNode,
CheckList,
CheckListItem.configure({
nested: true