Open platform to collect and prioritize feedback
Possible better search query
Matt Robertsd13cebc6955b
diff --git a/app/services/sqlstore/postgres/post.go b/app/services/sqlstore/postgres/post.goindex 05d072509..19f4af7fb 100644--- a/app/services/sqlstore/postgres/post.go+++ b/app/services/sqlstore/postgres/post.go@@ -478,24 +478,23 @@ func searchPosts(ctx context.Context, q *query.SearchPosts) error {tsConfig := MapLocaleToTSConfig(tenant.Locale)- // regexp_replace: replaces spaces with ' & ' for AND search; ':*' enables prefix matching in PostgreSQL full-text search- // query := fmt.Sprintf("to_tsquery('%s', regexp_replace($3, '\\s+', ' & ', 'g') || ':*')", tsConfig)- // tsVector := fmt.Sprintf("to_tsvector('%s', title) || ' ' || to_tsvector('%s', description)", tsConfig, tsConfig)-- tsVector := fmt.Sprintf("to_tsvector('%s', q.title) || ' ' || to_tsvector('%s', q.description) || ' ' || to_tsvector('simple', q.title) || ' ' || to_tsvector('simple', q.description) as vector", tsConfig, tsConfig)-- // if tsConfig != "english" {- // tsVector = fmt.Sprintf("%s || to_tsvector('english', title) || ' ' || to_tsvector('english', description)", tsVector)- // }+ // Build tsvector with multiple configurations for better multilingual support+ tsVector := fmt.Sprintf("to_tsvector('%s', q.title) || ' ' || to_tsvector('%s', q.description)", tsConfig, tsConfig)+ if tsConfig != "simple" {+ tsVector = fmt.Sprintf("%s || to_tsvector('simple', q.title) || ' ' || to_tsvector('simple', q.description)", tsVector)+ }+ tsVector = fmt.Sprintf("(%s) as vector", tsVector)- // if tsConfig != "simple" {- // tsVector = fmt.Sprintf("%s || to_tsvector('simple', title) || ' ' || to_tsvector('simple', description)", tsVector)- // }+ // Build tsquery with AND operator between words and prefix matching on each word+ // regexp_replace adds ':*' to each word for prefix matching, then joins with ' & ' for AND+ tsQueryExpr := fmt.Sprintf("to_tsquery('%s', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))", tsConfig)+ tsQuerySimple := "to_tsquery('simple', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))"- // tsVector = fmt.Sprintf("(%s)", tsVector)- score := fmt.Sprintf("ts_rank(vector, websearch_to_tsquery('%s', $3)) + ts_rank(vector, websearch_to_tsquery('simple', $3))", tsConfig)+ // Use ts_rank_cd (cover density ranking) for better relevance scoring+ score := fmt.Sprintf("ts_rank_cd(vector, %s) + ts_rank_cd(vector, %s)", tsQueryExpr, tsQuerySimple)- whereParts := fmt.Sprintf(`vector @@ websearch_to_tsquery('%s', $3) OR vector @@ websearch_to_tsquery('simple', $3)`, tsConfig)+ // Match against both locale-specific and simple configurations+ whereParts := fmt.Sprintf(`vector @@ %s OR vector @@ %s`, tsQueryExpr, tsQuerySimple)sql := fmt.Sprintf(`SELECT * FROM (SELECT %s, * FROM (%s) AS q ) as q2diff --git a/app/services/sqlstore/postgres/post_test.go b/app/services/sqlstore/postgres/post_test.goindex be33db099..1fe9f2c81 100644--- a/app/services/sqlstore/postgres/post_test.go+++ b/app/services/sqlstore/postgres/post_test.go@@ -87,13 +87,159 @@ func TestPostStorage_SearchGermanPosts(t *testing.T) {_, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Abwesenheiten Vertretungsplan', 'abwesenheiten-vertretungsplan', 7, 'final description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)Expect(err).IsNil()+ // Search for "Vertretung" - with German stemming, should match all posts containing Vertretung* variantsallPosts := &query.SearchPosts{Query: "Vertretung"}err = bus.Dispatch(germanTenantCtx, allPosts)Expect(err).IsNil()- Expect(allPosts.Result).HasLen(3)+ // All 7 posts contain "Vertretung" or variants like "Vertretungsplan", "Vertretungsplanung", etc.+ // German stemming correctly identifies these as the same root word+ Expect(allPosts.Result).HasLen(7)}+func TestPostStorage_SearchEnglishPosts_SingleWord(t *testing.T) {+ SetupDatabaseTest(t)+ defer TeardownDatabaseTest()++ now := time.Now()++ // Create posts with various forms of "integration"+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Add Twitter Integration', 'add-twitter-integration', 1, 'Would be great to integrate with Twitter', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('GitHub Integration Needed', 'github-integration-needed', 2, 'Please add GitHub integration', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Integrate with Slack', 'integrate-with-slack', 3, 'Slack integration would be useful', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Email Notifications', 'email-notifications', 4, 'Add email notification support', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ // Search for "integration" - should match posts 1, 2, and 3 (with stemming, "integrate" = "integration")+ searchIntegration := &query.SearchPosts{Query: "integration"}+ err = bus.Dispatch(demoTenantCtx, searchIntegration)+ Expect(err).IsNil()+ Expect(searchIntegration.Result).HasLen(3)++ // Verify the results contain the expected posts (should match all variations due to stemming)+ slugs := map[string]bool{}+ for _, post := range searchIntegration.Result {+ slugs[post.Slug] = true+ }+ Expect(slugs["add-twitter-integration"]).IsTrue()+ Expect(slugs["github-integration-needed"]).IsTrue()+ Expect(slugs["integrate-with-slack"]).IsTrue()+}++func TestPostStorage_SearchEnglishPosts_MultiWord(t *testing.T) {+ SetupDatabaseTest(t)+ defer TeardownDatabaseTest()++ now := time.Now()++ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Dark Mode Support', 'dark-mode-support', 1, 'Add dark mode to the application', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Light Theme Customization', 'light-theme-customization', 2, 'Customize the light theme colors', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Dark Theme Colors', 'dark-theme-colors', 3, 'Change dark theme color scheme', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('User Profile', 'user-profile', 4, 'Improve user profile page', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ searchDarkTheme := &query.SearchPosts{Query: "dark theme"}+ err = bus.Dispatch(demoTenantCtx, searchDarkTheme)+ Expect(err).IsNil()++ Expect(searchDarkTheme.Result).HasLen(3)+}++func TestPostStorage_SearchEnglishPosts_PartialMatch(t *testing.T) {+ SetupDatabaseTest(t)+ defer TeardownDatabaseTest()++ now := time.Now()++ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Authentication System', 'authentication-system', 1, 'Improve authentication', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Authorization Rules', 'authorization-rules', 2, 'Add better authorization', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('OAuth Support', 'oauth-support', 3, 'Support OAuth providers', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ // Search for "auth" - should match posts 1 and 2 (prefix matching)+ searchAuth := &query.SearchPosts{Query: "auth"}+ err = bus.Dispatch(demoTenantCtx, searchAuth)+ Expect(err).IsNil()+ Expect(searchAuth.Result).HasLen(2)++ slugs := map[string]bool{}+ for _, post := range searchAuth.Result {+ slugs[post.Slug] = true+ }+ Expect(slugs["authentication-system"]).IsTrue()+ Expect(slugs["authorization-rules"]).IsTrue()+}++func TestPostStorage_SearchEnglishPosts_CaseInsensitive(t *testing.T) {+ SetupDatabaseTest(t)+ defer TeardownDatabaseTest()++ now := time.Now()++ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('API Documentation', 'api-documentation', 1, 'Improve API docs', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Rest API Endpoints', 'rest-api-endpoints', 2, 'Add more REST API endpoints', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ // Search with different cases - all should return same results+ searchLower := &query.SearchPosts{Query: "api"}+ searchUpper := &query.SearchPosts{Query: "API"}+ searchMixed := &query.SearchPosts{Query: "Api"}++ err = bus.Dispatch(demoTenantCtx, searchLower, searchUpper, searchMixed)+ Expect(err).IsNil()++ Expect(searchLower.Result).HasLen(2)+ Expect(searchUpper.Result).HasLen(2)+ Expect(searchMixed.Result).HasLen(2)+}++func TestPostStorage_SearchEnglishPosts_DescriptionMatch(t *testing.T) {+ SetupDatabaseTest(t)+ defer TeardownDatabaseTest()++ now := time.Now()++ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Feature Request', 'feature-request-1', 1, 'Add support for exporting data to CSV format', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Another Feature', 'feature-request-2', 2, 'Improve the dashboard layout', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Export Functionality', 'export-functionality', 3, 'Export reports in PDF format', $1, 1, 1, 0)", now)+ Expect(err).IsNil()++ // Search for "export" - should match posts 1 and 3 (one in description, one in title)+ searchExport := &query.SearchPosts{Query: "export"}+ err = bus.Dispatch(demoTenantCtx, searchExport)+ Expect(err).IsNil()+ Expect(searchExport.Result).HasLen(2)++ slugs := map[string]bool{}+ for _, post := range searchExport.Result {+ slugs[post.Slug] = true+ }+ Expect(slugs["feature-request-1"]).IsTrue()+ Expect(slugs["export-functionality"]).IsTrue()+}+func TestPostStorage_AddAndGet(t *testing.T) {SetupDatabaseTest(t)defer TeardownDatabaseTest()