Skip to content

There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.

fix(server): avoid get object content when syncing (#9402)

liuyi378db1054be4
diff --git a/packages/backend/server/src/base/event/def.ts b/packages/backend/server/src/base/event/def.ts
index 7f253d9343..71d0e24176 100644
--- a/packages/backend/server/src/base/event/def.ts
+++ b/packages/backend/server/src/base/event/def.ts
@@ -24,6 +24,10 @@ export interface WorkspaceEvents {
workspaceId: Workspace['id'];
key: string;
}>;
+ sync: Payload<{
+ workspaceId: Workspace['id'];
+ key: string;
+ }>;
};
}
diff --git a/packages/backend/server/src/base/storage/providers/fs.ts b/packages/backend/server/src/base/storage/providers/fs.ts
index 41e17f3e66..081e678aca 100644
--- a/packages/backend/server/src/base/storage/providers/fs.ts
+++ b/packages/backend/server/src/base/storage/providers/fs.ts
@@ -61,6 +61,15 @@ export class FsStorageProvider implements StorageProvider {
this.logger.verbose(`Object \`${key}\` put`);
}
+ async head(key: string) {
+ const metadata = this.readMetadata(key);
+ if (!metadata) {
+ this.logger.verbose(`Object \`${key}\` not found`);
+ return undefined;
+ }
+ return metadata;
+ }
+
async get(key: string): Promise<{
body?: Readable;
metadata?: GetObjectMetadata;
diff --git a/packages/backend/server/src/base/storage/providers/provider.ts b/packages/backend/server/src/base/storage/providers/provider.ts
index 5b2bad24b2..06d8009773 100644
--- a/packages/backend/server/src/base/storage/providers/provider.ts
+++ b/packages/backend/server/src/base/storage/providers/provider.ts
@@ -34,6 +34,7 @@ export interface StorageProvider {
body: BlobInputType,
metadata?: PutObjectMetadata
): Promise<void>;
+ head(key: string): Promise<GetObjectMetadata | undefined>;
get(
key: string
): Promise<{ body?: BlobOutputType; metadata?: GetObjectMetadata }>;
diff --git a/packages/backend/server/src/core/storage/wrappers/blob.ts b/packages/backend/server/src/core/storage/wrappers/blob.ts
index 2ca900af67..85e4d50ad1 100644
--- a/packages/backend/server/src/core/storage/wrappers/blob.ts
+++ b/packages/backend/server/src/core/storage/wrappers/blob.ts
@@ -32,7 +32,7 @@ export class WorkspaceBlobStorage {
const meta: PutObjectMetadata = autoMetadata(blob);
await this.provider.put(`${workspaceId}/${key}`, blob, meta);
- this.trySyncBlobMeta(workspaceId, key, {
+ await this.upsert(workspaceId, key, {
contentType: meta.contentType ?? 'application/octet-stream',
contentLength: blob.length,
lastModified: new Date(),
@@ -119,53 +119,48 @@ export class WorkspaceBlobStorage {
private trySyncBlobsMeta(workspaceId: string, blobs: ListObjectsMetadata[]) {
for (const blob of blobs) {
- this.trySyncBlobMeta(workspaceId, blob.key);
+ this.event.emit('workspace.blob.sync', {
+ workspaceId,
+ key: blob.key,
+ });
}
}
- private trySyncBlobMeta(
+ private async upsert(
workspaceId: string,
key: string,
- meta?: GetObjectMetadata
+ meta: GetObjectMetadata
) {
- setImmediate(() => {
- this.syncBlobMeta(workspaceId, key, meta).catch(() => {
- /* never throw */
- });
+ await this.db.blob.upsert({
+ where: {
+ workspaceId_key: {
+ workspaceId,
+ key,
+ },
+ },
+ update: {
+ mime: meta.contentType,
+ size: meta.contentLength,
+ },
+ create: {
+ workspaceId,
+ key,
+ mime: meta.contentType,
+ size: meta.contentLength,
+ },
});
}
- private async syncBlobMeta(
- workspaceId: string,
- key: string,
- meta?: GetObjectMetadata
- ) {
+ @OnEvent('workspace.blob.sync')
+ async syncBlobMeta({
+ workspaceId,
+ key,
+ }: EventPayload<'workspace.blob.sync'>) {
try {
- if (!meta) {
- const blob = await this.get(workspaceId, key);
- meta = blob.metadata;
- blob.body?.destroy();
- }
+ const meta = await this.provider.head(`${workspaceId}/${key}`);
if (meta) {
- await this.db.blob.upsert({
- where: {
- workspaceId_key: {
- workspaceId,
- key,
- },
- },
- update: {
- mime: meta.contentType,
- size: meta.contentLength,
- },
- create: {
- workspaceId,
- key,
- mime: meta.contentType,
- size: meta.contentLength,
- },
- });
+ await this.upsert(workspaceId, key, meta);
} else {
await this.db.blob.deleteMany({
where: {
diff --git a/packages/backend/server/src/plugins/storage/providers/s3.ts b/packages/backend/server/src/plugins/storage/providers/s3.ts
index 2dbb819b36..2b4d693cf2 100644
--- a/packages/backend/server/src/plugins/storage/providers/s3.ts
+++ b/packages/backend/server/src/plugins/storage/providers/s3.ts
@@ -1,9 +1,10 @@
-/* eslint-disable @typescript-eslint/no-non-null-assertion */
+/* oxlint-disable @typescript-eslint/no-non-null-assertion */
import { Readable } from 'node:stream';
import {
DeleteObjectCommand,
GetObjectCommand,
+ HeadObjectCommand,
ListObjectsV2Command,
NoSuchKey,
PutObjectCommand,
@@ -75,6 +76,34 @@ export class S3StorageProvider implements StorageProvider {
}
}
+ async head(key: string) {
+ try {
+ const obj = await this.client.send(
+ new HeadObjectCommand({
+ Bucket: this.bucket,
+ Key: key,
+ })
+ );
+
+ return {
+ contentType: obj.ContentType!,
+ contentLength: obj.ContentLength!,
+ lastModified: obj.LastModified!,
+ checksumCRC32: obj.ChecksumCRC32,
+ };
+ } catch (e) {
+ // 404
+ if (e instanceof NoSuchKey) {
+ this.logger.verbose(`Object \`${key}\` not found`);
+ return undefined;
+ } else {
+ throw new Error(`Failed to head object \`${key}\``, {
+ cause: e,
+ });
+ }
+ }
+ }
+
async get(key: string): Promise<{
body?: Readable;
metadata?: GetObjectMetadata;