Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
fix(filter): match memos without the flag in JSON bool comparisons (#6382)
Suliman Abdulrazzaq0c53b775ce9a
diff --git a/filter/README.md b/filter/README.mdindex cc05072d4..c52c57b1d 100644--- a/filter/README.md+++ b/filter/README.md@@ -59,8 +59,10 @@ stmt, _ := engine.CompileToStatement(ctx, `has_task_list && visibility == "PUBLImatching are case-sensitive, without Unicode normalization; `%` and `_` instring operands are literal. Hierarchy works through implied ancestors alreadypresent in the memo tag set, not through prefix matching.-- **Boolean Flags** — Fields such as `has_task_list` render as `IS TRUE` equality- checks, or comparisons against `CAST('true' AS JSON)` depending on the dialect.+- **Boolean Flags** — Fields such as `has_task_list` render as an `IS TRUE` check+ (a comparison against `CAST('true' AS JSON)` on MySQL). Memo payloads omit false+ flags, so a missing key counts as false: `flag == false` and `flag != true`+ render as the negated check and match the same memos as `!flag`.- **Presence Flags** — `has_location` renders as a JSON key-existence check on`memo.payload` (`$.location`). A missing key and an explicit JSON null bothcount as absent on every dialect; any other value — including an empty object —diff --git a/filter/engine_test.go b/filter/engine_test.goindex a7b27a0e6..575487b48 100644--- a/filter/engine_test.go+++ b/filter/engine_test.go@@ -665,6 +665,96 @@ func TestHasLocationSQLiteBehavior(t *testing.T) {}}+func TestRenderJSONBoolComparisonsPerDialect(t *testing.T) {+ t.Parallel()++ engine, err := NewEngine(NewSchema())+ require.NoError(t, err)++ predicates := map[DialectName]string{+ DialectSQLite: "JSON_EXTRACT(`memo`.`payload`, '$.property.hasLink') IS TRUE",+ DialectMySQL: "COALESCE(JSON_EXTRACT(`memo`.`payload`, '$.property.hasLink'), CAST('false' AS JSON)) = CAST('true' AS JSON)",+ DialectPostgres: "(memo.payload->'property'->>'hasLink')::boolean IS TRUE",+ }+ for dialect, predicate := range predicates {+ cases := []struct {+ expr string+ sql string+ }{+ {`has_link`, predicate},+ {`!has_link`, "NOT (" + predicate + ")"},+ {`has_link == true`, predicate},+ {`has_link == false`, "NOT (" + predicate + ")"},+ {`has_link != true`, "NOT (" + predicate + ")"},+ {`has_link != false`, predicate},+ }+ for _, tc := range cases {+ stmt, err := engine.CompileToStatement(context.Background(), tc.expr, RenderOptions{Dialect: dialect})+ require.NoError(t, err, dialect, tc.expr)+ require.Equal(t, tc.sql, stmt.SQL, dialect, tc.expr)+ require.Empty(t, stmt.Args, dialect, tc.expr)+ }+ }+}++// TestJSONBoolComparisonSQLiteBehavior pins the comparison semantics of the+// JSON boolean flags against a real database. Memo payloads omit false flags,+// so a missing key, a missing property object and a NULL payload all count as+// false, and `flag == false` must match the same rows as `!flag`.+func TestJSONBoolComparisonSQLiteBehavior(t *testing.T) {+ db, err := sql.Open("sqlite", ":memory:")+ require.NoError(t, err)+ db.SetMaxOpenConns(1)+ t.Cleanup(func() { require.NoError(t, db.Close()) })++ _, err = db.Exec(`CREATE TABLE memo (id INTEGER PRIMARY KEY, payload TEXT)`)+ require.NoError(t, err)++ engine, err := NewEngine(NewSchema())+ require.NoError(t, err)++ flags := map[string]string{+ "has_task_list": "hasTaskList",+ "has_link": "hasLink",+ "has_code": "hasCode",+ "has_incomplete_tasks": "hasIncompleteTasks",+ }+ for field, key := range flags {+ _, err = db.Exec(`DELETE FROM memo`)+ require.NoError(t, err)+ for _, fixture := range []struct {+ id int+ payload any+ }{+ {1, `{}`},+ {2, fmt.Sprintf(`{"property":{"%s":true}}`, key)},+ {3, fmt.Sprintf(`{"property":{"%s":false}}`, key)},+ {4, `{"property":{}}`},+ {5, nil},+ } {+ _, err = db.Exec(`INSERT INTO memo (id, payload) VALUES (?, ?)`, fixture.id, fixture.payload)+ require.NoError(t, err)+ }++ cases := []struct {+ expr string+ want []int+ }{+ {field, []int{2}},+ {"!" + field, []int{1, 3, 4, 5}},+ {field + " == true", []int{2}},+ {field + " == false", []int{1, 3, 4, 5}},+ {field + " != true", []int{1, 3, 4, 5}},+ {field + " != false", []int{2}},+ }+ for _, tc := range cases {+ stmt, err := engine.CompileToStatement(context.Background(), tc.expr, RenderOptions{Dialect: DialectSQLite})+ require.NoError(t, err, tc.expr)+ require.Equal(t, tc.want, selectMemoIDs(t, db, stmt), tc.expr)+ }+ }+}+func TestCompileRejectsOversizedExpression(t *testing.T) {t.Parallel()engine, err := NewEngine(NewSchema())diff --git a/filter/render.go b/filter/render.goindex a2bae487a..4aeddb1f9 100644--- a/filter/render.go+++ b/filter/render.go@@ -362,59 +362,20 @@ func (r *renderer) renderBoolColumnComparison(field Field, op ComparisonOperator}, nil}+// renderJSONBoolComparison compares a JSON boolean flag with a literal. Memo+// payloads omit false flags, so a missing key means false: comparisons reuse+// jsonBoolPredicate, which already treats a missing key or a NULL payload as+// false on every dialect.func (r *renderer) renderJSONBoolComparison(field Field, op ComparisonOperator, right ValueExpr) (renderResult, error) {value, err := expectBool(right)if err != nil {- return renderResult{}, err+ return renderResult{}, errors.Wrap(err, "json boolean comparison requires a boolean value")}-- jsonExpr := jsonExtractExpr(r.dialect, field)- switch r.dialect {- case DialectSQLite:- switch op {- case CompareEq:- if field.Name == "has_task_list" {- target := "0"- if value {- target = "1"- }- return renderResult{sql: fmt.Sprintf("%s = %s", jsonExpr, target)}, nil- }- if value {- return renderResult{sql: fmt.Sprintf("%s IS TRUE", jsonExpr)}, nil- }- return renderResult{sql: fmt.Sprintf("NOT(%s IS TRUE)", jsonExpr)}, nil- case CompareNeq:- if field.Name == "has_task_list" {- target := "0"- if value {- target = "1"- }- return renderResult{sql: fmt.Sprintf("%s != %s", jsonExpr, target)}, nil- }- if value {- return renderResult{sql: fmt.Sprintf("NOT(%s IS TRUE)", jsonExpr)}, nil- }- return renderResult{sql: fmt.Sprintf("%s IS TRUE", jsonExpr)}, nil- default:- return renderResult{}, errors.Errorf("operator %s not supported for boolean JSON field", op)- }- case DialectMySQL:- boolStr := "false"- if value {- boolStr = "true"- }- return renderResult{- sql: fmt.Sprintf("%s %s CAST('%s' AS JSON)", jsonExpr, sqlOperator(op), boolStr),- }, nil- case DialectPostgres:- placeholder := r.addArg(value)- return renderResult{- sql: fmt.Sprintf("(%s)::boolean %s %s", jsonExpr, sqlOperator(op), placeholder),- }, nil- default:- return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect)+ predicate, err := r.jsonBoolPredicate(field)+ if err != nil {+ return renderResult{}, errors.Wrap(err, "failed to render JSON boolean comparison")}+ return renderPredicateComparison(field, op, value, predicate)}func (r *renderer) renderInCondition(cond *InCondition) (renderResult, error) {@@ -762,6 +723,13 @@ func (r *renderer) renderJSONExistsComparison(field Field, op ComparisonOperatorif err != nil {return renderResult{}, errors.Wrap(err, "failed to render JSON existence comparison")}+ return renderPredicateComparison(field, op, value, existsSQL)+}++// renderPredicateComparison renders `field == value` or `field != value` for a+// field whose truth is fully described by predicate: the predicate itself when+// the comparison asks for true, its negation when it asks for false.+func renderPredicateComparison(field Field, op ComparisonOperator, value bool, predicate string) (renderResult, error) {want := valueswitch op {case CompareEq:@@ -771,9 +739,9 @@ func (r *renderer) renderJSONExistsComparison(field Field, op ComparisonOperatorreturn renderResult{}, errors.Errorf("operator %s not supported for field %q", op, field.Name)}if want {- return renderResult{sql: existsSQL}, nil+ return renderResult{sql: predicate}, nil}- return renderResult{sql: fmt.Sprintf("NOT (%s)", existsSQL)}, nil+ return renderResult{sql: fmt.Sprintf("NOT (%s)", predicate)}, nil}func (r *renderer) jsonBoolPredicate(field Field) (string, error) {diff --git a/store/test/memo_filter_test.go b/store/test/memo_filter_test.goindex 0c99525ef..c867e029f 100644--- a/store/test/memo_filter_test.go+++ b/store/test/memo_filter_test.go@@ -603,8 +603,15 @@ func TestMemoFilterHasTaskList(t *testing.T) {memos = tc.ListWithFilter(`has_task_list == true`)require.Len(t, memos, 1)- // Note: has_task_list == false is not tested because JSON boolean fields- // with false value may not be queryable when the field is not present in JSON+ // Test: has_task_list == false matches memos whose payload omits the flag+ memos = tc.ListWithFilter(`has_task_list == false`)+ require.Len(t, memos, 1)+ require.False(t, memos[0].Payload.GetProperty().GetHasTaskList())++ // Test: has_task_list != true+ memos = tc.ListWithFilter(`has_task_list != true`)+ require.Len(t, memos, 1)+ require.False(t, memos[0].Payload.GetProperty().GetHasTaskList())}func TestMemoFilterHasLink(t *testing.T) {@@ -620,6 +627,21 @@ func TestMemoFilterHasLink(t *testing.T) {memos := tc.ListWithFilter(`has_link`)require.Len(t, memos, 1)require.True(t, memos[0].Payload.Property.HasLink)++ // Test: has_link == false matches memos whose payload omits the flag+ memos = tc.ListWithFilter(`has_link == false`)+ require.Len(t, memos, 1)+ require.False(t, memos[0].Payload.GetProperty().GetHasLink())++ // Test: has_link != true+ memos = tc.ListWithFilter(`has_link != true`)+ require.Len(t, memos, 1)+ require.False(t, memos[0].Payload.GetProperty().GetHasLink())++ // Test: has_link != false+ memos = tc.ListWithFilter(`has_link != false`)+ require.Len(t, memos, 1)+ require.True(t, memos[0].Payload.GetProperty().GetHasLink())}func TestMemoFilterHasCode(t *testing.T) {@@ -635,6 +657,11 @@ func TestMemoFilterHasCode(t *testing.T) {memos := tc.ListWithFilter(`has_code`)require.Len(t, memos, 1)require.True(t, memos[0].Payload.Property.HasCode)++ // Test: has_code == false matches memos whose payload omits the flag+ memos = tc.ListWithFilter(`has_code == false`)+ require.Len(t, memos, 1)+ require.False(t, memos[0].Payload.GetProperty().GetHasCode())}func TestMemoFilterHasIncompleteTasks(t *testing.T) {@@ -658,6 +685,11 @@ func TestMemoFilterHasIncompleteTasks(t *testing.T) {memos := tc.ListWithFilter(`has_incomplete_tasks`)require.Len(t, memos, 1)require.True(t, memos[0].Payload.Property.HasIncompleteTasks)++ // Test: has_incomplete_tasks == false matches memos whose payload omits the flag+ memos = tc.ListWithFilter(`has_incomplete_tasks == false`)+ require.Len(t, memos, 1)+ require.False(t, memos[0].Payload.GetProperty().GetHasIncompleteTasks())}func TestMemoFilterCombinedJSONBool(t *testing.T) {