Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
feat: implement attachment filtering functionality
Johnny78aa41336ab1
diff --git a/plugin/filter/engine.go b/plugin/filter/engine.goindex 25b5485f9..5ee068a66 100644--- a/plugin/filter/engine.go+++ b/plugin/filter/engine.go@@ -96,9 +96,12 @@ func (p *Program) Render(opts RenderOptions) (Statement, error) {}var (- defaultOnce sync.Once- defaultInst *Engine- defaultErr error+ defaultOnce sync.Once+ defaultInst *Engine+ defaultErr error+ defaultAttachmentOnce sync.Once+ defaultAttachmentInst *Engine+ defaultAttachmentErr error)// DefaultEngine returns the process-wide memo filter engine.@@ -109,6 +112,14 @@ func DefaultEngine() (*Engine, error) {return defaultInst, defaultErr}+// DefaultAttachmentEngine returns the process-wide attachment filter engine.+func DefaultAttachmentEngine() (*Engine, error) {+ defaultAttachmentOnce.Do(func() {+ defaultAttachmentInst, defaultAttachmentErr = NewEngine(NewAttachmentSchema())+ })+ return defaultAttachmentInst, defaultAttachmentErr+}+func normalizeLegacyFilter(expr string) string {expr = rewriteNumericLogicalOperand(expr, "&&")expr = rewriteNumericLogicalOperand(expr, "||")diff --git a/plugin/filter/schema.go b/plugin/filter/schema.goindex 4d5e3b4dc..21fae7423 100644--- a/plugin/filter/schema.go+++ b/plugin/filter/schema.go@@ -243,6 +243,62 @@ func NewSchema() Schema {}}+// NewAttachmentSchema constructs the attachment filter schema and CEL environment.+func NewAttachmentSchema() Schema {+ fields := map[string]Field{+ "filename": {+ Name: "filename",+ Kind: FieldKindScalar,+ Type: FieldTypeString,+ Column: Column{Table: "resource", Name: "filename"},+ SupportsContains: true,+ Expressions: map[DialectName]string{},+ },+ "mime_type": {+ Name: "mime_type",+ Kind: FieldKindScalar,+ Type: FieldTypeString,+ Column: Column{Table: "resource", Name: "type"},+ Expressions: map[DialectName]string{},+ },+ "create_time": {+ Name: "create_time",+ Kind: FieldKindScalar,+ Type: FieldTypeTimestamp,+ Column: Column{Table: "resource", Name: "created_ts"},+ Expressions: map[DialectName]string{+ DialectMySQL: "UNIX_TIMESTAMP(%s)",+ DialectPostgres: "EXTRACT(EPOCH FROM TO_TIMESTAMP(%s))",+ },+ },+ "memo": {+ Name: "memo",+ Kind: FieldKindScalar,+ Type: FieldTypeString,+ Column: Column{Table: "resource", Name: "memo_uid"},+ Expressions: map[DialectName]string{},+ AllowedComparisonOps: map[ComparisonOperator]bool{+ CompareEq: true,+ CompareNeq: true,+ },+ },+ }++ envOptions := []cel.EnvOption{+ cel.Variable("filename", cel.StringType),+ cel.Variable("mime_type", cel.StringType),+ cel.Variable("create_time", cel.IntType),+ cel.Variable("memo", cel.StringType),+ nowFunction,+ }++ return Schema{+ Name: "attachment",+ Fields: fields,+ EnvOptions: envOptions,+ }+}+// columnExpr returns the field expression for the given dialect, applying// any schema-specific overrides (e.g. UNIX timestamp conversions).func (f Field) columnExpr(d DialectName) string {diff --git a/proto/api/v1/attachment_service.proto b/proto/api/v1/attachment_service.protoindex ad2631f45..7385e114b 100644--- a/proto/api/v1/attachment_service.proto+++ b/proto/api/v1/attachment_service.proto@@ -101,9 +101,9 @@ message ListAttachmentsRequest {string page_token = 2 [(google.api.field_behavior) = OPTIONAL];// Optional. Filter to apply to the list results.- // Example: "type=image/png" or "filename:*.jpg"- // Supported operators: =, !=, <, <=, >, >=, :- // Supported fields: filename, type, size, create_time, memo+ // Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"+ // Supported operators: =, !=, <, <=, >, >=, : (contains), in+ // Supported fields: filename, mime_type, create_time, memostring filter = 3 [(google.api.field_behavior) = OPTIONAL];// Optional. The order to sort results by.diff --git a/proto/gen/api/v1/attachment_service.pb.go b/proto/gen/api/v1/attachment_service.pb.goindex 535bbb3bd..ea6525e5c 100644--- a/proto/gen/api/v1/attachment_service.pb.go+++ b/proto/gen/api/v1/attachment_service.pb.go@@ -201,9 +201,9 @@ type ListAttachmentsRequest struct {// Provide this to retrieve the subsequent page.PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"`// Optional. Filter to apply to the list results.- // Example: "type=image/png" or "filename:*.jpg"- // Supported operators: =, !=, <, <=, >, >=, :- // Supported fields: filename, type, size, create_time, memo+ // Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"+ // Supported operators: =, !=, <, <=, >, >=, : (contains), in+ // Supported fields: filename, mime_type, create_time, memoFilter string `protobuf:"bytes,3,opt,name=filter,proto3" json:"filter,omitempty"`// Optional. The order to sort results by.// Example: "create_time desc" or "filename asc"diff --git a/proto/gen/openapi.yaml b/proto/gen/openapi.yamlindex 1e95422e6..f4746dbd5 100644--- a/proto/gen/openapi.yaml+++ b/proto/gen/openapi.yaml@@ -97,9 +97,9 @@ paths:in: querydescription: |-Optional. Filter to apply to the list results.- Example: "type=image/png" or "filename:*.jpg"- Supported operators: =, !=, <, <=, >, >=, :- Supported fields: filename, type, size, create_time, memo+ Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"+ Supported operators: =, !=, <, <=, >, >=, : (contains), in+ Supported fields: filename, mime_type, create_time, memoschema:type: string- name: orderBydiff --git a/server/router/api/v1/attachment_service.go b/server/router/api/v1/attachment_service.goindex 2e05d4526..642fcb67f 100644--- a/server/router/api/v1/attachment_service.go+++ b/server/router/api/v1/attachment_service.go@@ -21,6 +21,7 @@ import ("github.com/usememos/memos/internal/profile""github.com/usememos/memos/internal/util"+ "github.com/usememos/memos/plugin/filter""github.com/usememos/memos/plugin/storage/s3"v1pb "github.com/usememos/memos/proto/gen/api/v1"storepb "github.com/usememos/memos/proto/gen/store"@@ -156,6 +157,14 @@ func (s *APIV1Service) ListAttachments(ctx context.Context, request *v1pb.ListAtOffset: &offset,}+ // Parse filter if provided+ if request.Filter != "" {+ if err := s.validateAttachmentFilter(ctx, request.Filter); err != nil {+ return nil, status.Errorf(codes.InvalidArgument, "invalid filter: %v", err)+ }+ findAttachment.Filters = append(findAttachment.Filters, request.Filter)+ }+attachments, err := s.Store.ListAttachments(ctx, findAttachment)if err != nil {return nil, status.Errorf(codes.Internal, "failed to list attachments: %v", err)@@ -472,3 +481,29 @@ func isValidMimeType(mimeType string) bool {matched, _ := regexp.MatchString(`^[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}/[a-zA-Z0-9][a-zA-Z0-9!#$&^_.+-]{0,126}$`, mimeType)return matched}++func (s *APIV1Service) validateAttachmentFilter(ctx context.Context, filterStr string) error {+ if filterStr == "" {+ return errors.New("filter cannot be empty")+ }++ engine, err := filter.DefaultAttachmentEngine()+ if err != nil {+ return err+ }++ var dialect filter.DialectName+ switch s.Profile.Driver {+ case "mysql":+ dialect = filter.DialectMySQL+ case "postgres":+ dialect = filter.DialectPostgres+ default:+ dialect = filter.DialectSQLite+ }++ if _, err := engine.CompileToStatement(ctx, filterStr, filter.RenderOptions{Dialect: dialect}); err != nil {+ return errors.Wrap(err, "failed to compile filter")+ }+ return nil+}diff --git a/store/attachment.go b/store/attachment.goindex 6b4dcba85..cd3ac2a16 100644--- a/store/attachment.go+++ b/store/attachment.go@@ -51,6 +51,7 @@ type FindAttachment struct {MemoIDList []int32HasRelatedMemo boolStorageType *storepb.AttachmentStorageType+ Filters []stringLimit *intOffset *int}diff --git a/store/db/mysql/attachment.go b/store/db/mysql/attachment.goindex 6eb59338d..ead254d88 100644--- a/store/db/mysql/attachment.go+++ b/store/db/mysql/attachment.go@@ -9,6 +9,7 @@ import ("github.com/pkg/errors""google.golang.org/protobuf/encoding/protojson"+ "github.com/usememos/memos/plugin/filter"storepb "github.com/usememos/memos/proto/gen/store""github.com/usememos/memos/store")@@ -83,6 +84,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String())}+ if len(find.Filters) > 0 {+ engine, err := filter.DefaultAttachmentEngine()+ if err != nil {+ return nil, errors.Wrap(err, "failed to get filter engine")+ }+ if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectMySQL, &where, &args); err != nil {+ return nil, errors.Wrap(err, "failed to append filter conditions")+ }+ }+fields := []string{"`resource`.`id` AS `id`","`resource`.`uid` AS `uid`",diff --git a/store/db/postgres/attachment.go b/store/db/postgres/attachment.goindex 1c17941d6..9ee970fbd 100644--- a/store/db/postgres/attachment.go+++ b/store/db/postgres/attachment.go@@ -9,6 +9,7 @@ import ("github.com/pkg/errors""google.golang.org/protobuf/encoding/protojson"+ "github.com/usememos/memos/plugin/filter"storepb "github.com/usememos/memos/proto/gen/store""github.com/usememos/memos/store")@@ -72,6 +73,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([where, args = append(where, "resource.storage_type = "+placeholder(len(args)+1)), append(args, v.String())}+ if len(find.Filters) > 0 {+ engine, err := filter.DefaultAttachmentEngine()+ if err != nil {+ return nil, errors.Wrap(err, "failed to get filter engine")+ }+ if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectPostgres, &where, &args); err != nil {+ return nil, errors.Wrap(err, "failed to append filter conditions")+ }+ }+fields := []string{"resource.id AS id","resource.uid AS uid",diff --git a/store/db/sqlite/attachment.go b/store/db/sqlite/attachment.goindex c35257338..04653b185 100644--- a/store/db/sqlite/attachment.go+++ b/store/db/sqlite/attachment.go@@ -9,6 +9,7 @@ import ("github.com/pkg/errors""google.golang.org/protobuf/encoding/protojson"+ "github.com/usememos/memos/plugin/filter"storepb "github.com/usememos/memos/proto/gen/store""github.com/usememos/memos/store")@@ -76,6 +77,16 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([where, args = append(where, "`resource`.`storage_type` = ?"), append(args, find.StorageType.String())}+ if len(find.Filters) > 0 {+ engine, err := filter.DefaultAttachmentEngine()+ if err != nil {+ return nil, errors.Wrap(err, "failed to get filter engine")+ }+ if err := filter.AppendConditions(ctx, engine, find.Filters, filter.DialectSQLite, &where, &args); err != nil {+ return nil, errors.Wrap(err, "failed to append filter conditions")+ }+ }+fields := []string{"`resource`.`id` AS `id`","`resource`.`uid` AS `uid`",diff --git a/store/test/attachment_test.go b/store/test/attachment_test.goindex 8655029bf..68e080636 100644--- a/store/test/attachment_test.go+++ b/store/test/attachment_test.go@@ -61,3 +61,62 @@ func TestAttachmentStore(t *testing.T) {require.ErrorContains(t, err, "attachment not found")ts.Close()}++func TestAttachmentStoreWithFilter(t *testing.T) {+ ctx := context.Background()+ ts := NewTestingStore(ctx, t)++ _, err := ts.CreateAttachment(ctx, &store.Attachment{+ UID: shortuuid.New(),+ CreatorID: 101,+ Filename: "test.png",+ Blob: []byte("test"),+ Type: "image/png",+ Size: 1000,+ })+ require.NoError(t, err)++ _, err = ts.CreateAttachment(ctx, &store.Attachment{+ UID: shortuuid.New(),+ CreatorID: 101,+ Filename: "test.jpg",+ Blob: []byte("test"),+ Type: "image/jpeg",+ Size: 2000,+ })+ require.NoError(t, err)++ _, err = ts.CreateAttachment(ctx, &store.Attachment{+ UID: shortuuid.New(),+ CreatorID: 101,+ Filename: "test.pdf",+ Blob: []byte("test"),+ Type: "application/pdf",+ Size: 3000,+ })+ require.NoError(t, err)++ attachments, err := ts.ListAttachments(ctx, &store.FindAttachment{+ CreatorID: &[]int32{101}[0],+ Filters: []string{`mime_type == "image/png"`},+ })+ require.NoError(t, err)+ require.Len(t, attachments, 1)+ require.Equal(t, "image/png", attachments[0].Type)++ attachments, err = ts.ListAttachments(ctx, &store.FindAttachment{+ CreatorID: &[]int32{101}[0],+ Filters: []string{`mime_type in ["image/png", "image/jpeg"]`},+ })+ require.NoError(t, err)+ require.Len(t, attachments, 2)++ attachments, err = ts.ListAttachments(ctx, &store.FindAttachment{+ CreatorID: &[]int32{101}[0],+ Filters: []string{`filename.contains("test")`},+ })+ require.NoError(t, err)+ require.Len(t, attachments, 3)++ ts.Close()+}diff --git a/web/src/types/proto/api/v1/attachment_service_pb.ts b/web/src/types/proto/api/v1/attachment_service_pb.tsindex 31d8cd58f..defda15c9 100644--- a/web/src/types/proto/api/v1/attachment_service_pb.ts+++ b/web/src/types/proto/api/v1/attachment_service_pb.ts@@ -139,9 +139,9 @@ export type ListAttachmentsRequest = Message<"memos.api.v1.ListAttachmentsReques/*** Optional. Filter to apply to the list results.- * Example: "type=image/png" or "filename:*.jpg"- * Supported operators: =, !=, <, <=, >, >=, :- * Supported fields: filename, type, size, create_time, memo+ * Example: "mime_type==\"image/png\"" or "filename.contains(\"test\")"+ * Supported operators: =, !=, <, <=, >, >=, : (contains), in+ * Supported fields: filename, mime_type, create_time, memo** @generated from field: string filter = 3;*/