Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

Possible better search query

Matt Robertsd13cebc6955b
diff --git a/app/services/sqlstore/postgres/post.go b/app/services/sqlstore/postgres/post.go
index 05d072509..19f4af7fb 100644
--- a/app/services/sqlstore/postgres/post.go
+++ b/app/services/sqlstore/postgres/post.go
@@ -478,24 +478,23 @@ func searchPosts(ctx context.Context, q *query.SearchPosts) error {
tsConfig := MapLocaleToTSConfig(tenant.Locale)
- // regexp_replace: replaces spaces with ' & ' for AND search; ':*' enables prefix matching in PostgreSQL full-text search
- // query := fmt.Sprintf("to_tsquery('%s', regexp_replace($3, '\\s+', ' & ', 'g') || ':*')", tsConfig)
- // tsVector := fmt.Sprintf("to_tsvector('%s', title) || ' ' || to_tsvector('%s', description)", tsConfig, tsConfig)
-
- tsVector := fmt.Sprintf("to_tsvector('%s', q.title) || ' ' || to_tsvector('%s', q.description) || ' ' || to_tsvector('simple', q.title) || ' ' || to_tsvector('simple', q.description) as vector", tsConfig, tsConfig)
-
- // if tsConfig != "english" {
- // tsVector = fmt.Sprintf("%s || to_tsvector('english', title) || ' ' || to_tsvector('english', description)", tsVector)
- // }
+ // Build tsvector with multiple configurations for better multilingual support
+ tsVector := fmt.Sprintf("to_tsvector('%s', q.title) || ' ' || to_tsvector('%s', q.description)", tsConfig, tsConfig)
+ if tsConfig != "simple" {
+ tsVector = fmt.Sprintf("%s || to_tsvector('simple', q.title) || ' ' || to_tsvector('simple', q.description)", tsVector)
+ }
+ tsVector = fmt.Sprintf("(%s) as vector", tsVector)
- // if tsConfig != "simple" {
- // tsVector = fmt.Sprintf("%s || to_tsvector('simple', title) || ' ' || to_tsvector('simple', description)", tsVector)
- // }
+ // Build tsquery with AND operator between words and prefix matching on each word
+ // regexp_replace adds ':*' to each word for prefix matching, then joins with ' & ' for AND
+ tsQueryExpr := fmt.Sprintf("to_tsquery('%s', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))", tsConfig)
+ tsQuerySimple := "to_tsquery('simple', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))"
- // tsVector = fmt.Sprintf("(%s)", tsVector)
- score := fmt.Sprintf("ts_rank(vector, websearch_to_tsquery('%s', $3)) + ts_rank(vector, websearch_to_tsquery('simple', $3))", tsConfig)
+ // Use ts_rank_cd (cover density ranking) for better relevance scoring
+ score := fmt.Sprintf("ts_rank_cd(vector, %s) + ts_rank_cd(vector, %s)", tsQueryExpr, tsQuerySimple)
- whereParts := fmt.Sprintf(`vector @@ websearch_to_tsquery('%s', $3) OR vector @@ websearch_to_tsquery('simple', $3)`, tsConfig)
+ // Match against both locale-specific and simple configurations
+ whereParts := fmt.Sprintf(`vector @@ %s OR vector @@ %s`, tsQueryExpr, tsQuerySimple)
sql := fmt.Sprintf(`
SELECT * FROM (SELECT %s, * FROM (%s) AS q ) as q2
diff --git a/app/services/sqlstore/postgres/post_test.go b/app/services/sqlstore/postgres/post_test.go
index be33db099..1fe9f2c81 100644
--- a/app/services/sqlstore/postgres/post_test.go
+++ b/app/services/sqlstore/postgres/post_test.go
@@ -87,13 +87,159 @@ func TestPostStorage_SearchGermanPosts(t *testing.T) {
_, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Abwesenheiten Vertretungsplan', 'abwesenheiten-vertretungsplan', 7, 'final description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)
Expect(err).IsNil()
+ // Search for "Vertretung" - with German stemming, should match all posts containing Vertretung* variants
allPosts := &query.SearchPosts{Query: "Vertretung"}
err = bus.Dispatch(germanTenantCtx, allPosts)
Expect(err).IsNil()
- Expect(allPosts.Result).HasLen(3)
+ // All 7 posts contain "Vertretung" or variants like "Vertretungsplan", "Vertretungsplanung", etc.
+ // German stemming correctly identifies these as the same root word
+ Expect(allPosts.Result).HasLen(7)
}
+func TestPostStorage_SearchEnglishPosts_SingleWord(t *testing.T) {
+ SetupDatabaseTest(t)
+ defer TeardownDatabaseTest()
+
+ now := time.Now()
+
+ // Create posts with various forms of "integration"
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Add Twitter Integration', 'add-twitter-integration', 1, 'Would be great to integrate with Twitter', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('GitHub Integration Needed', 'github-integration-needed', 2, 'Please add GitHub integration', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Integrate with Slack', 'integrate-with-slack', 3, 'Slack integration would be useful', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Email Notifications', 'email-notifications', 4, 'Add email notification support', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ // Search for "integration" - should match posts 1, 2, and 3 (with stemming, "integrate" = "integration")
+ searchIntegration := &query.SearchPosts{Query: "integration"}
+ err = bus.Dispatch(demoTenantCtx, searchIntegration)
+ Expect(err).IsNil()
+ Expect(searchIntegration.Result).HasLen(3)
+
+ // Verify the results contain the expected posts (should match all variations due to stemming)
+ slugs := map[string]bool{}
+ for _, post := range searchIntegration.Result {
+ slugs[post.Slug] = true
+ }
+ Expect(slugs["add-twitter-integration"]).IsTrue()
+ Expect(slugs["github-integration-needed"]).IsTrue()
+ Expect(slugs["integrate-with-slack"]).IsTrue()
+}
+
+func TestPostStorage_SearchEnglishPosts_MultiWord(t *testing.T) {
+ SetupDatabaseTest(t)
+ defer TeardownDatabaseTest()
+
+ now := time.Now()
+
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Dark Mode Support', 'dark-mode-support', 1, 'Add dark mode to the application', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Light Theme Customization', 'light-theme-customization', 2, 'Customize the light theme colors', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Dark Theme Colors', 'dark-theme-colors', 3, 'Change dark theme color scheme', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('User Profile', 'user-profile', 4, 'Improve user profile page', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ searchDarkTheme := &query.SearchPosts{Query: "dark theme"}
+ err = bus.Dispatch(demoTenantCtx, searchDarkTheme)
+ Expect(err).IsNil()
+
+ Expect(searchDarkTheme.Result).HasLen(3)
+}
+
+func TestPostStorage_SearchEnglishPosts_PartialMatch(t *testing.T) {
+ SetupDatabaseTest(t)
+ defer TeardownDatabaseTest()
+
+ now := time.Now()
+
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Authentication System', 'authentication-system', 1, 'Improve authentication', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Authorization Rules', 'authorization-rules', 2, 'Add better authorization', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('OAuth Support', 'oauth-support', 3, 'Support OAuth providers', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ // Search for "auth" - should match posts 1 and 2 (prefix matching)
+ searchAuth := &query.SearchPosts{Query: "auth"}
+ err = bus.Dispatch(demoTenantCtx, searchAuth)
+ Expect(err).IsNil()
+ Expect(searchAuth.Result).HasLen(2)
+
+ slugs := map[string]bool{}
+ for _, post := range searchAuth.Result {
+ slugs[post.Slug] = true
+ }
+ Expect(slugs["authentication-system"]).IsTrue()
+ Expect(slugs["authorization-rules"]).IsTrue()
+}
+
+func TestPostStorage_SearchEnglishPosts_CaseInsensitive(t *testing.T) {
+ SetupDatabaseTest(t)
+ defer TeardownDatabaseTest()
+
+ now := time.Now()
+
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('API Documentation', 'api-documentation', 1, 'Improve API docs', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Rest API Endpoints', 'rest-api-endpoints', 2, 'Add more REST API endpoints', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ // Search with different cases - all should return same results
+ searchLower := &query.SearchPosts{Query: "api"}
+ searchUpper := &query.SearchPosts{Query: "API"}
+ searchMixed := &query.SearchPosts{Query: "Api"}
+
+ err = bus.Dispatch(demoTenantCtx, searchLower, searchUpper, searchMixed)
+ Expect(err).IsNil()
+
+ Expect(searchLower.Result).HasLen(2)
+ Expect(searchUpper.Result).HasLen(2)
+ Expect(searchMixed.Result).HasLen(2)
+}
+
+func TestPostStorage_SearchEnglishPosts_DescriptionMatch(t *testing.T) {
+ SetupDatabaseTest(t)
+ defer TeardownDatabaseTest()
+
+ now := time.Now()
+
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Feature Request', 'feature-request-1', 1, 'Add support for exporting data to CSV format', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Another Feature', 'feature-request-2', 2, 'Improve the dashboard layout', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Export Functionality', 'export-functionality', 3, 'Export reports in PDF format', $1, 1, 1, 0)", now)
+ Expect(err).IsNil()
+
+ // Search for "export" - should match posts 1 and 3 (one in description, one in title)
+ searchExport := &query.SearchPosts{Query: "export"}
+ err = bus.Dispatch(demoTenantCtx, searchExport)
+ Expect(err).IsNil()
+ Expect(searchExport.Result).HasLen(2)
+
+ slugs := map[string]bool{}
+ for _, post := range searchExport.Result {
+ slugs[post.Slug] = true
+ }
+ Expect(slugs["feature-request-1"]).IsTrue()
+ Expect(slugs["export-functionality"]).IsTrue()
+}
+
func TestPostStorage_AddAndGet(t *testing.T) {
SetupDatabaseTest(t)
defer TeardownDatabaseTest()