Open platform to collect and prioritize feedback
Querying with the new search column 💪
Matt Roberts9caf19291151
diff --git a/app/services/sqlstore/postgres/post.go b/app/services/sqlstore/postgres/post.goindex ff86f0fb9..1a294bd83 100644--- a/app/services/sqlstore/postgres/post.go+++ b/app/services/sqlstore/postgres/post.go@@ -25,13 +25,13 @@ import ()type dbPost struct {- Vector []byte `db:"vector"`ID int `db:"id"`Number int `db:"number"`Title string `db:"title"`Slug string `db:"slug"`Description string `db:"description"`CreatedAt time.Time `db:"created_at"`+ Search []byte `db:"search"`User *dbUser `db:"user"`HasVoted bool `db:"has_voted"`VotesCount int `db:"votes_count"`@@ -122,16 +122,17 @@ var (WHERE posts.tenant_id = $1GROUP BY post_id)- SELECT p.id,- p.number,- p.title,- p.slug,- p.description,+ SELECT p.id,+ p.number,+ p.title,+ p.slug,+ p.description,p.created_at,+ p.search,COALESCE(agg_s.all, 0) as votes_count,COALESCE(agg_c.all, 0) as comments_count,COALESCE(agg_s.recent, 0) AS recent_votes_count,- COALESCE(agg_c.recent, 0) AS recent_comments_count,+ COALESCE(agg_c.recent, 0) AS recent_comments_count,p.status,u.id AS user_id,u.name AS user_name,@@ -452,29 +453,24 @@ func findSimilarPosts(ctx context.Context, q *query.FindSimilarPosts) error {} else {tsConfig := MapLocaleToTSConfig(tenant.Locale)- tsVector := fmt.Sprintf("to_tsvector('%s', q.title) || ' ' || to_tsvector('%s', q.description)", tsConfig, tsConfig)- if tsConfig != "simple" {- tsVector = fmt.Sprintf("%s || to_tsvector('simple', q.title) || ' ' || to_tsvector('simple', q.description)", tsVector)- }- tsVector = fmt.Sprintf("(%s) as vector", tsVector)-// Build tsquery with AND operator between words and prefix matching on each word- // regexp_replace adds ':*' to each word for prefix matching, then joins with ' & ' for AND+ // The search column already contains both language-specific and simple tsvectorstsQueryExpr := fmt.Sprintf("to_tsquery('%s', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))", tsConfig)tsQuerySimple := "to_tsquery('simple', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))"// Use ts_rank_cd (cover density ranking) for better relevance scoring- score := fmt.Sprintf("ts_rank_cd(vector, %s) + ts_rank_cd(vector, %s)", tsQueryExpr, tsQuerySimple)+ // Query against the generated search column which combines language-specific and simple tsvectors+ score := fmt.Sprintf("ts_rank_cd(q.search, %s) + ts_rank_cd(q.search, %s)", tsQueryExpr, tsQuerySimple)- // Match against both locale-specific and simple configurations- whereParts := fmt.Sprintf(`vector @@ %s OR vector @@ %s`, tsQueryExpr, tsQuerySimple)+ // Match against the pre-computed search column+ whereParts := fmt.Sprintf(`q.search @@ %s OR q.search @@ %s`, tsQueryExpr, tsQuerySimple)sql := fmt.Sprintf(`- SELECT * FROM (SELECT %s, * FROM (%s) AS q ) as q2+ SELECT * FROM (%s) AS qWHERE %sORDER BY %s DESCLIMIT 5- `, tsVector, innerQuery, whereParts, score)+ `, innerQuery, whereParts, score)err = trx.Select(&posts, sql, tenant.ID, pq.Array([]enum.PostStatus{enum.PostOpen,enum.PostStarted,@@ -526,30 +522,19 @@ func searchPosts(ctx context.Context, q *query.SearchPosts) error {tsConfig := MapLocaleToTSConfig(tenant.Locale)- // Build tsvector with multiple configurations for better multilingual support- tsVector := fmt.Sprintf("to_tsvector('%s', q.title) || ' ' || to_tsvector('%s', q.description)", tsConfig, tsConfig)- if tsConfig != "simple" {- tsVector = fmt.Sprintf("%s || to_tsvector('simple', q.title) || ' ' || to_tsvector('simple', q.description)", tsVector)- }- tsVector = fmt.Sprintf("(%s) as vector", tsVector)-- // Build tsquery with AND operator between words and prefix matching on each word- // regexp_replace adds ':*' to each word for prefix matching, then joins with ' & ' for ANDtsQueryExpr := fmt.Sprintf("to_tsquery('%s', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))", tsConfig)tsQuerySimple := "to_tsquery('simple', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))"- // Use ts_rank_cd (cover density ranking) for better relevance scoring- score := fmt.Sprintf("ts_rank_cd(vector, %s) + ts_rank_cd(vector, %s)", tsQueryExpr, tsQuerySimple)+ score := fmt.Sprintf("ts_rank_cd(q.search, %s) + ts_rank_cd(q.search, %s)", tsQueryExpr, tsQuerySimple)- // Match against both locale-specific and simple configurations- whereParts := fmt.Sprintf(`vector @@ %s OR vector @@ %s`, tsQueryExpr, tsQuerySimple)+ whereParts := fmt.Sprintf(`q.search @@ %s OR q.search @@ %s`, tsQueryExpr, tsQuerySimple)sql := fmt.Sprintf(`- SELECT * FROM (SELECT %s, * FROM (%s) AS q ) as q2+ SELECT * FROM (%s) AS qWHERE %sORDER BY %s DESCLIMIT %s- `, tsVector, innerQuery, whereParts, score, q.Limit)+ `, innerQuery, whereParts, score, q.Limit)err = trx.Select(&posts, sql, tenant.ID, pq.Array([]enum.PostStatus{enum.PostOpen,enum.PostStarted,diff --git a/app/services/sqlstore/postgres/post_test.go b/app/services/sqlstore/postgres/post_test.goindex 1fe9f2c81..ba5d7db40 100644--- a/app/services/sqlstore/postgres/post_test.go+++ b/app/services/sqlstore/postgres/post_test.go@@ -24,10 +24,10 @@ func TestPostStorage_GetAll(t *testing.T) {now := time.Now()- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('add twitter integration', 'add-twitter-integration', 1, 'Would be great to see it integrated with twitter', $1, 1, 1, 1)", now)+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('add twitter integration', 'add-twitter-integration', 1, 'Would be great to see it integrated with twitter', $1, 1, 1, 1, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('this is my post', 'this-is-my-post', 2, 'no description', $1, 1, 2, 2)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('this is my post', 'this-is-my-post', 2, 'no description', $1, 1, 2, 2, 'english')", now)Expect(err).IsNil()allPosts := &query.GetAllPosts{}@@ -66,25 +66,25 @@ func TestPostStorage_SearchGermanPosts(t *testing.T) {now := time.Now()- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Wann kommt der \"Wächter\" für das neue Stunden- und Vertretungsplanmodul?', 'wann-kommt-der-wachter-fur-das-neue-stunden-und-vertretungsplanmodul', 1, 'Description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Wann kommt der \"Wächter\" für das neue Stunden- und Vertretungsplanmodul?', 'wann-kommt-der-wachter-fur-das-neue-stunden-und-vertretungsplanmodul', 1, 'Description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Neuer Stunden/ Vertretungsplanwächter', 'neuer-stunden-vertretungsplanwachter', 2, 'no description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Neuer Stunden/ Vertretungsplanwächter', 'neuer-stunden-vertretungsplanwachter', 2, 'no description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Ungeklärte Vertretung wird zu Entfall', 'ungeklarte-vertretung-wird-zu-entfall', 3, 'some description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Ungeklärte Vertretung wird zu Entfall', 'ungeklarte-vertretung-wird-zu-entfall', 3, 'some description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Vertretungsplan drucken', 'vertretungsplan-drucken', 4, 'another description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Vertretungsplan drucken', 'vertretungsplan-drucken', 4, 'another description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Vertretungsplanung Wochenansicht', 'vertretungsplanung-wochenansicht', 5, 'description here', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Vertretungsplanung Wochenansicht', 'vertretungsplanung-wochenansicht', 5, 'description here', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('VertretungsBoard', 'vertretungsboard', 6, 'board description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('VertretungsBoard', 'vertretungsboard', 6, 'board description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Abwesenheiten Vertretungsplan', 'abwesenheiten-vertretungsplan', 7, 'final description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Abwesenheiten Vertretungsplan', 'abwesenheiten-vertretungsplan', 7, 'final description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)Expect(err).IsNil()// Search for "Vertretung" - with German stemming, should match all posts containing Vertretung* variants@@ -104,16 +104,16 @@ func TestPostStorage_SearchEnglishPosts_SingleWord(t *testing.T) {now := time.Now()// Create posts with various forms of "integration"- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Add Twitter Integration', 'add-twitter-integration', 1, 'Would be great to integrate with Twitter', $1, 1, 1, 0)", now)+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Add Twitter Integration', 'add-twitter-integration', 1, 'Would be great to integrate with Twitter', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('GitHub Integration Needed', 'github-integration-needed', 2, 'Please add GitHub integration', $1, 1, 1, 0)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('GitHub Integration Needed', 'github-integration-needed', 2, 'Please add GitHub integration', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Integrate with Slack', 'integrate-with-slack', 3, 'Slack integration would be useful', $1, 1, 1, 0)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Integrate with Slack', 'integrate-with-slack', 3, 'Slack integration would be useful', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Email Notifications', 'email-notifications', 4, 'Add email notification support', $1, 1, 1, 0)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Email Notifications', 'email-notifications', 4, 'Add email notification support', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()// Search for "integration" - should match posts 1, 2, and 3 (with stemming, "integrate" = "integration")@@ -138,16 +138,16 @@ func TestPostStorage_SearchEnglishPosts_MultiWord(t *testing.T) {now := time.Now()- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Dark Mode Support', 'dark-mode-support', 1, 'Add dark mode to the application', $1, 1, 1, 0)", now)+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Dark Mode Support', 'dark-mode-support', 1, 'Add dark mode to the application', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Light Theme Customization', 'light-theme-customization', 2, 'Customize the light theme colors', $1, 1, 1, 0)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Light Theme Customization', 'light-theme-customization', 2, 'Customize the light theme colors', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Dark Theme Colors', 'dark-theme-colors', 3, 'Change dark theme color scheme', $1, 1, 1, 0)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Dark Theme Colors', 'dark-theme-colors', 3, 'Change dark theme color scheme', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('User Profile', 'user-profile', 4, 'Improve user profile page', $1, 1, 1, 0)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('User Profile', 'user-profile', 4, 'Improve user profile page', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()searchDarkTheme := &query.SearchPosts{Query: "dark theme"}@@ -163,13 +163,13 @@ func TestPostStorage_SearchEnglishPosts_PartialMatch(t *testing.T) {now := time.Now()- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Authentication System', 'authentication-system', 1, 'Improve authentication', $1, 1, 1, 0)", now)+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Authentication System', 'authentication-system', 1, 'Improve authentication', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Authorization Rules', 'authorization-rules', 2, 'Add better authorization', $1, 1, 1, 0)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Authorization Rules', 'authorization-rules', 2, 'Add better authorization', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('OAuth Support', 'oauth-support', 3, 'Support OAuth providers', $1, 1, 1, 0)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('OAuth Support', 'oauth-support', 3, 'Support OAuth providers', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()// Search for "auth" - should match posts 1 and 2 (prefix matching)@@ -192,10 +192,10 @@ func TestPostStorage_SearchEnglishPosts_CaseInsensitive(t *testing.T) {now := time.Now()- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('API Documentation', 'api-documentation', 1, 'Improve API docs', $1, 1, 1, 0)", now)+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('API Documentation', 'api-documentation', 1, 'Improve API docs', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Rest API Endpoints', 'rest-api-endpoints', 2, 'Add more REST API endpoints', $1, 1, 1, 0)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Rest API Endpoints', 'rest-api-endpoints', 2, 'Add more REST API endpoints', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()// Search with different cases - all should return same results@@ -217,13 +217,13 @@ func TestPostStorage_SearchEnglishPosts_DescriptionMatch(t *testing.T) {now := time.Now()- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Feature Request', 'feature-request-1', 1, 'Add support for exporting data to CSV format', $1, 1, 1, 0)", now)+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Feature Request', 'feature-request-1', 1, 'Add support for exporting data to CSV format', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Another Feature', 'feature-request-2', 2, 'Improve the dashboard layout', $1, 1, 1, 0)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Another Feature', 'feature-request-2', 2, 'Improve the dashboard layout', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Export Functionality', 'export-functionality', 3, 'Export reports in PDF format', $1, 1, 1, 0)", now)+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Export Functionality', 'export-functionality', 3, 'Export reports in PDF format', $1, 1, 1, 0, 'english')", now)Expect(err).IsNil()// Search for "export" - should match posts 1 and 3 (one in description, one in title)