Skip to content

armgit / fider

Public Imported from GitHub
0

Open platform to collect and prioritize feedback

Querying with the new search column 💪

Matt Roberts9caf19291151
diff --git a/app/services/sqlstore/postgres/post.go b/app/services/sqlstore/postgres/post.go
index ff86f0fb9..1a294bd83 100644
--- a/app/services/sqlstore/postgres/post.go
+++ b/app/services/sqlstore/postgres/post.go
@@ -25,13 +25,13 @@ import (
)
type dbPost struct {
- Vector []byte `db:"vector"`
ID int `db:"id"`
Number int `db:"number"`
Title string `db:"title"`
Slug string `db:"slug"`
Description string `db:"description"`
CreatedAt time.Time `db:"created_at"`
+ Search []byte `db:"search"`
User *dbUser `db:"user"`
HasVoted bool `db:"has_voted"`
VotesCount int `db:"votes_count"`
@@ -122,16 +122,17 @@ var (
WHERE posts.tenant_id = $1
GROUP BY post_id
)
- SELECT p.id,
- p.number,
- p.title,
- p.slug,
- p.description,
+ SELECT p.id,
+ p.number,
+ p.title,
+ p.slug,
+ p.description,
p.created_at,
+ p.search,
COALESCE(agg_s.all, 0) as votes_count,
COALESCE(agg_c.all, 0) as comments_count,
COALESCE(agg_s.recent, 0) AS recent_votes_count,
- COALESCE(agg_c.recent, 0) AS recent_comments_count,
+ COALESCE(agg_c.recent, 0) AS recent_comments_count,
p.status,
u.id AS user_id,
u.name AS user_name,
@@ -452,29 +453,24 @@ func findSimilarPosts(ctx context.Context, q *query.FindSimilarPosts) error {
} else {
tsConfig := MapLocaleToTSConfig(tenant.Locale)
- tsVector := fmt.Sprintf("to_tsvector('%s', q.title) || ' ' || to_tsvector('%s', q.description)", tsConfig, tsConfig)
- if tsConfig != "simple" {
- tsVector = fmt.Sprintf("%s || to_tsvector('simple', q.title) || ' ' || to_tsvector('simple', q.description)", tsVector)
- }
- tsVector = fmt.Sprintf("(%s) as vector", tsVector)
-
// Build tsquery with AND operator between words and prefix matching on each word
- // regexp_replace adds ':*' to each word for prefix matching, then joins with ' & ' for AND
+ // The search column already contains both language-specific and simple tsvectors
tsQueryExpr := fmt.Sprintf("to_tsquery('%s', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))", tsConfig)
tsQuerySimple := "to_tsquery('simple', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))"
// Use ts_rank_cd (cover density ranking) for better relevance scoring
- score := fmt.Sprintf("ts_rank_cd(vector, %s) + ts_rank_cd(vector, %s)", tsQueryExpr, tsQuerySimple)
+ // Query against the generated search column which combines language-specific and simple tsvectors
+ score := fmt.Sprintf("ts_rank_cd(q.search, %s) + ts_rank_cd(q.search, %s)", tsQueryExpr, tsQuerySimple)
- // Match against both locale-specific and simple configurations
- whereParts := fmt.Sprintf(`vector @@ %s OR vector @@ %s`, tsQueryExpr, tsQuerySimple)
+ // Match against the pre-computed search column
+ whereParts := fmt.Sprintf(`q.search @@ %s OR q.search @@ %s`, tsQueryExpr, tsQuerySimple)
sql := fmt.Sprintf(`
- SELECT * FROM (SELECT %s, * FROM (%s) AS q ) as q2
+ SELECT * FROM (%s) AS q
WHERE %s
ORDER BY %s DESC
LIMIT 5
- `, tsVector, innerQuery, whereParts, score)
+ `, innerQuery, whereParts, score)
err = trx.Select(&posts, sql, tenant.ID, pq.Array([]enum.PostStatus{
enum.PostOpen,
enum.PostStarted,
@@ -526,30 +522,19 @@ func searchPosts(ctx context.Context, q *query.SearchPosts) error {
tsConfig := MapLocaleToTSConfig(tenant.Locale)
- // Build tsvector with multiple configurations for better multilingual support
- tsVector := fmt.Sprintf("to_tsvector('%s', q.title) || ' ' || to_tsvector('%s', q.description)", tsConfig, tsConfig)
- if tsConfig != "simple" {
- tsVector = fmt.Sprintf("%s || to_tsvector('simple', q.title) || ' ' || to_tsvector('simple', q.description)", tsVector)
- }
- tsVector = fmt.Sprintf("(%s) as vector", tsVector)
-
- // Build tsquery with AND operator between words and prefix matching on each word
- // regexp_replace adds ':*' to each word for prefix matching, then joins with ' & ' for AND
tsQueryExpr := fmt.Sprintf("to_tsquery('%s', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))", tsConfig)
tsQuerySimple := "to_tsquery('simple', regexp_replace(regexp_replace($3, '\\\\s+', ':* & ', 'g'), '$', ':*'))"
- // Use ts_rank_cd (cover density ranking) for better relevance scoring
- score := fmt.Sprintf("ts_rank_cd(vector, %s) + ts_rank_cd(vector, %s)", tsQueryExpr, tsQuerySimple)
+ score := fmt.Sprintf("ts_rank_cd(q.search, %s) + ts_rank_cd(q.search, %s)", tsQueryExpr, tsQuerySimple)
- // Match against both locale-specific and simple configurations
- whereParts := fmt.Sprintf(`vector @@ %s OR vector @@ %s`, tsQueryExpr, tsQuerySimple)
+ whereParts := fmt.Sprintf(`q.search @@ %s OR q.search @@ %s`, tsQueryExpr, tsQuerySimple)
sql := fmt.Sprintf(`
- SELECT * FROM (SELECT %s, * FROM (%s) AS q ) as q2
+ SELECT * FROM (%s) AS q
WHERE %s
ORDER BY %s DESC
LIMIT %s
- `, tsVector, innerQuery, whereParts, score, q.Limit)
+ `, innerQuery, whereParts, score, q.Limit)
err = trx.Select(&posts, sql, tenant.ID, pq.Array([]enum.PostStatus{
enum.PostOpen,
enum.PostStarted,
diff --git a/app/services/sqlstore/postgres/post_test.go b/app/services/sqlstore/postgres/post_test.go
index 1fe9f2c81..ba5d7db40 100644
--- a/app/services/sqlstore/postgres/post_test.go
+++ b/app/services/sqlstore/postgres/post_test.go
@@ -24,10 +24,10 @@ func TestPostStorage_GetAll(t *testing.T) {
now := time.Now()
- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('add twitter integration', 'add-twitter-integration', 1, 'Would be great to see it integrated with twitter', $1, 1, 1, 1)", now)
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('add twitter integration', 'add-twitter-integration', 1, 'Would be great to see it integrated with twitter', $1, 1, 1, 1, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('this is my post', 'this-is-my-post', 2, 'no description', $1, 1, 2, 2)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('this is my post', 'this-is-my-post', 2, 'no description', $1, 1, 2, 2, 'english')", now)
Expect(err).IsNil()
allPosts := &query.GetAllPosts{}
@@ -66,25 +66,25 @@ func TestPostStorage_SearchGermanPosts(t *testing.T) {
now := time.Now()
- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Wann kommt der \"Wächter\" für das neue Stunden- und Vertretungsplanmodul?', 'wann-kommt-der-wachter-fur-das-neue-stunden-und-vertretungsplanmodul', 1, 'Description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Wann kommt der \"Wächter\" für das neue Stunden- und Vertretungsplanmodul?', 'wann-kommt-der-wachter-fur-das-neue-stunden-und-vertretungsplanmodul', 1, 'Description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Neuer Stunden/ Vertretungsplanwächter', 'neuer-stunden-vertretungsplanwachter', 2, 'no description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Neuer Stunden/ Vertretungsplanwächter', 'neuer-stunden-vertretungsplanwachter', 2, 'no description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Ungeklärte Vertretung wird zu Entfall', 'ungeklarte-vertretung-wird-zu-entfall', 3, 'some description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Ungeklärte Vertretung wird zu Entfall', 'ungeklarte-vertretung-wird-zu-entfall', 3, 'some description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Vertretungsplan drucken', 'vertretungsplan-drucken', 4, 'another description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Vertretungsplan drucken', 'vertretungsplan-drucken', 4, 'another description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Vertretungsplanung Wochenansicht', 'vertretungsplanung-wochenansicht', 5, 'description here', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Vertretungsplanung Wochenansicht', 'vertretungsplanung-wochenansicht', 5, 'description here', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('VertretungsBoard', 'vertretungsboard', 6, 'board description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('VertretungsBoard', 'vertretungsboard', 6, 'board description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Abwesenheiten Vertretungsplan', 'abwesenheiten-vertretungsplan', 7, 'final description', $1, $2, $3, 0)", now, germanTenant.ID, germanJonSnow.ID)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Abwesenheiten Vertretungsplan', 'abwesenheiten-vertretungsplan', 7, 'final description', $1, $2, $3, 0, 'german')", now, germanTenant.ID, germanJonSnow.ID)
Expect(err).IsNil()
// Search for "Vertretung" - with German stemming, should match all posts containing Vertretung* variants
@@ -104,16 +104,16 @@ func TestPostStorage_SearchEnglishPosts_SingleWord(t *testing.T) {
now := time.Now()
// Create posts with various forms of "integration"
- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Add Twitter Integration', 'add-twitter-integration', 1, 'Would be great to integrate with Twitter', $1, 1, 1, 0)", now)
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Add Twitter Integration', 'add-twitter-integration', 1, 'Would be great to integrate with Twitter', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('GitHub Integration Needed', 'github-integration-needed', 2, 'Please add GitHub integration', $1, 1, 1, 0)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('GitHub Integration Needed', 'github-integration-needed', 2, 'Please add GitHub integration', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Integrate with Slack', 'integrate-with-slack', 3, 'Slack integration would be useful', $1, 1, 1, 0)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Integrate with Slack', 'integrate-with-slack', 3, 'Slack integration would be useful', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Email Notifications', 'email-notifications', 4, 'Add email notification support', $1, 1, 1, 0)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Email Notifications', 'email-notifications', 4, 'Add email notification support', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
// Search for "integration" - should match posts 1, 2, and 3 (with stemming, "integrate" = "integration")
@@ -138,16 +138,16 @@ func TestPostStorage_SearchEnglishPosts_MultiWord(t *testing.T) {
now := time.Now()
- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Dark Mode Support', 'dark-mode-support', 1, 'Add dark mode to the application', $1, 1, 1, 0)", now)
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Dark Mode Support', 'dark-mode-support', 1, 'Add dark mode to the application', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Light Theme Customization', 'light-theme-customization', 2, 'Customize the light theme colors', $1, 1, 1, 0)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Light Theme Customization', 'light-theme-customization', 2, 'Customize the light theme colors', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Dark Theme Colors', 'dark-theme-colors', 3, 'Change dark theme color scheme', $1, 1, 1, 0)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Dark Theme Colors', 'dark-theme-colors', 3, 'Change dark theme color scheme', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('User Profile', 'user-profile', 4, 'Improve user profile page', $1, 1, 1, 0)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('User Profile', 'user-profile', 4, 'Improve user profile page', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
searchDarkTheme := &query.SearchPosts{Query: "dark theme"}
@@ -163,13 +163,13 @@ func TestPostStorage_SearchEnglishPosts_PartialMatch(t *testing.T) {
now := time.Now()
- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Authentication System', 'authentication-system', 1, 'Improve authentication', $1, 1, 1, 0)", now)
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Authentication System', 'authentication-system', 1, 'Improve authentication', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Authorization Rules', 'authorization-rules', 2, 'Add better authorization', $1, 1, 1, 0)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Authorization Rules', 'authorization-rules', 2, 'Add better authorization', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('OAuth Support', 'oauth-support', 3, 'Support OAuth providers', $1, 1, 1, 0)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('OAuth Support', 'oauth-support', 3, 'Support OAuth providers', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
// Search for "auth" - should match posts 1 and 2 (prefix matching)
@@ -192,10 +192,10 @@ func TestPostStorage_SearchEnglishPosts_CaseInsensitive(t *testing.T) {
now := time.Now()
- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('API Documentation', 'api-documentation', 1, 'Improve API docs', $1, 1, 1, 0)", now)
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('API Documentation', 'api-documentation', 1, 'Improve API docs', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Rest API Endpoints', 'rest-api-endpoints', 2, 'Add more REST API endpoints', $1, 1, 1, 0)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Rest API Endpoints', 'rest-api-endpoints', 2, 'Add more REST API endpoints', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
// Search with different cases - all should return same results
@@ -217,13 +217,13 @@ func TestPostStorage_SearchEnglishPosts_DescriptionMatch(t *testing.T) {
now := time.Now()
- _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Feature Request', 'feature-request-1', 1, 'Add support for exporting data to CSV format', $1, 1, 1, 0)", now)
+ _, err := trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Feature Request', 'feature-request-1', 1, 'Add support for exporting data to CSV format', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Another Feature', 'feature-request-2', 2, 'Improve the dashboard layout', $1, 1, 1, 0)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Another Feature', 'feature-request-2', 2, 'Improve the dashboard layout', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
- _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status) VALUES ('Export Functionality', 'export-functionality', 3, 'Export reports in PDF format', $1, 1, 1, 0)", now)
+ _, err = trx.Execute("INSERT INTO posts (title, slug, number, description, created_at, tenant_id, user_id, status, language) VALUES ('Export Functionality', 'export-functionality', 3, 'Export reports in PDF format', $1, 1, 1, 0, 'english')", now)
Expect(err).IsNil()
// Search for "export" - should match posts 1 and 3 (one in description, one in title)