Self hosted FLOSS fitness/workout, nutrition and weight tracker
Use a simple trigram search
Roland Geiderc590299e56b2
diff --git a/wger/exercises/api/views.py b/wger/exercises/api/views.pyindex 3ed69fe64..16b1eaf02 100644--- a/wger/exercises/api/views.py+++ b/wger/exercises/api/views.py@@ -17,18 +17,17 @@import loggingfrom uuid import UUID+# Third Party+import bleach+from actstream import action as actstream_action+from bleach.css_sanitizer import CSSSanitizer# Djangofrom django.conf import settings-from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector+from django.contrib.postgres.search import TrigramSimilarityfrom django.db.models import Qfrom django.utils.decorators import method_decoratorfrom django.utils.translation import gettext as _from django.views.decorators.cache import cache_page--# Third Party-import bleach-from actstream import action as actstream_action-from bleach.css_sanitizer import CSSSanitizerfrom drf_spectacular.types import OpenApiTypesfrom drf_spectacular.utils import (OpenApiParameter,@@ -100,6 +99,7 @@ class ExerciseBaseViewSet(ModelViewSet):For a read-only endpoint with all the information of an exercise, see /api/v2/exercisebaseinfo/"""+queryset = ExerciseBase.translations.all()serializer_class = ExerciseBaseSerializerpermission_classes = (CanContributeExercises,)@@ -149,6 +149,7 @@ class ExerciseTranslationViewSet(ModelViewSet):"""API endpoint for editing or adding exercise translation objects."""+queryset = Exercise.objects.all()permission_classes = (CanContributeExercises,)serializer_class = ExerciseTranslationSerializer@@ -350,16 +351,14 @@ def search(request):# Postgres uses a full-text searchif is_postgres_db():- vector = SearchVector('search_column')- search_query = SearchQuery(q, search_type="websearch")- query = query.annotate(rank=SearchRank(vector, search_query)) \- .filter(search_column=search_query) \- .order_by('-rank', 'name')+ query = (+ query.annotate(similarity=TrigramSimilarity("name", q))+ .filter(Q(similarity__gt=0.15) | Q(alias__alias__icontains=q))+ .order_by("-similarity", "name")+ )else:query = query.filter(Q(name__icontains=q) | Q(alias__alias__icontains=q))- print(query.query)-for translation in query:image = Nonethumbnail = None@@ -385,7 +384,7 @@ def search(request):}}results.append(result_json)- response['suggestions'] = results+ response["suggestions"] = resultsreturn Response(response)diff --git a/wger/exercises/migrations/0029_full_text_search.py b/wger/exercises/migrations/0029_full_text_search.pyindex 174fbf422..3ba780cf9 100644--- a/wger/exercises/migrations/0029_full_text_search.py+++ b/wger/exercises/migrations/0029_full_text_search.py@@ -1,99 +1,29 @@# Generated by Django 4.2.6 on 2024-01-18 16:55import django.contrib.postgres.search+from django.contrib.postgres.operations import TrigramExtension, BtreeGinExtensionfrom django.db import migrations, models-def forwards(apps, schema_editor):- if not schema_editor.connection.vendor.startswith('postgres'):- return-- schema_editor.execute('''- CREATE OR REPLACE FUNCTION update_exercise_fulltext_index() RETURNS trigger AS $$- DECLARE- aliases TEXT;- known_langs TEXT[];- BEGIN- SELECT ARRAY(SELECT cfgname FROM pg_ts_config) INTO known_langs;-- IF NEW.language_id IS NOT NULL THEN- SELECT lower(full_name_en)- INTO NEW.index_language- FROM core_language- WHERE id = NEW.language_id;-- -- If the language is not known, just use English- IF NEW.index_language IS NULL OR NEW.index_language NOT IN (SELECT unnest(known_langs)) THEN- NEW.index_language = 'english';- END IF;- ELSE- -- If language_id is NULL, just set to English- NEW.index_language = 'english';- END IF;-- SELECT COALESCE(string_agg(a.alias, ' '), '')- INTO aliases- FROM exercises_exercise AS t- LEFT JOIN exercises_alias AS a- ON t.id = a.exercise_id- WHERE a.exercise_id = NEW.id;-- NEW.search_column :=- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(new.name, '')), 'A') ||- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(aliases, '')), 'A') ||- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(new.description, '')), 'D');-- RETURN NEW;- END- $$ LANGUAGE plpgsql;-- -- Create the trigger- CREATE OR REPLACE TRIGGER exercise_search_column_trigger- BEFORE INSERT OR UPDATE- ON exercises_exercise- FOR EACH ROW EXECUTE FUNCTION- update_exercise_fulltext_index();-- -- Force an update of the search index- UPDATE exercises_exercise SET name = name WHERE true;- ''')---def backwards(apps, schema_editor):- if not schema_editor.connection.vendor.startswith('postgres'):- return-- schema_editor.execute('''- DROP TRIGGER IF EXISTS exercise_search_column_trigger ON exercises_exercise;- DROP FUNCTION IF EXISTS update_exercise_fulltext_index();- ''')--class Migration(migrations.Migration):dependencies = [('exercises', '0028_add_uuid_alias_and_comments'),]operations = [- migrations.AddField(- model_name='exercise',- name='index_language',- field=models.CharField(default='english', editable=False, max_length=30),- ),- migrations.AddField(- model_name='exercise',- name='search_column',- field=django.contrib.postgres.search.SearchVectorField(editable=False, null=True),- ),- migrations.AddField(- model_name='historicalexercise',- name='index_language',- field=models.CharField(default='english', editable=False, max_length=30),+ BtreeGinExtension(),+ TrigramExtension(),+ migrations.AddIndex(+ model_name="exercise",+ index=django.contrib.postgres.indexes.GinIndex(+ fields=["name"],+ name="exercises_e_name_ac11f4_gin",+ ),),- migrations.AddField(- model_name='historicalexercise',- name='search_column',- field=django.contrib.postgres.search.SearchVectorField(editable=False, null=True),+ migrations.AddIndex(+ model_name="alias",+ index=django.contrib.postgres.indexes.GinIndex(+ fields=["alias"], name="exercises_a_alias_227e38_gin"+ ),),- migrations.RunPython(forwards, backwards)]diff --git a/wger/exercises/models/exercise.py b/wger/exercises/models/exercise.pyindex d769d9534..0ca2a8e9b 100644--- a/wger/exercises/models/exercise.py+++ b/wger/exercises/models/exercise.py@@ -18,7 +18,7 @@import uuid# Django-from django.contrib.postgres.search import SearchVectorField+from django.contrib.postgres.indexes import GinIndexfrom django.core.validators import MinLengthValidatorfrom django.db import modelsfrom django.urls import reverse@@ -96,19 +96,6 @@ class Exercise(AbstractLicenseModel, AbstractHistoryMixin, models.Model):)""" Refers to the base exercise with non translated information """- search_column = SearchVectorField(- null=True,- editable=False,- )- """Column used for full text search"""-- index_language = models.CharField(- default='english',- editable=False,- max_length=30,- )- """The language used for the full text search"""-history = HistoricalRecords()"""Edit history"""@@ -120,6 +107,7 @@ class Exercise(AbstractLicenseModel, AbstractHistoryMixin, models.Model):ordering = ["name",]+ indexes = (GinIndex(fields=["name"]),)def get_absolute_url(self):"""diff --git a/wger/exercises/models/exercise_alias.py b/wger/exercises/models/exercise_alias.pyindex 9ff823a53..e470222a4 100644--- a/wger/exercises/models/exercise_alias.py+++ b/wger/exercises/models/exercise_alias.py@@ -18,6 +18,7 @@import uuid# Django+from django.contrib.postgres.indexes import GinIndexfrom django.db import modelsfrom django.utils.translation import gettext_lazy as _@@ -38,6 +39,7 @@ class Alias(models.Model):"""Model for an exercise (name)alias"""+uuid = models.UUIDField(default=uuid.uuid4,editable=False,@@ -60,6 +62,9 @@ class Alias(models.Model):history = HistoricalRecords()"""Edit history"""+ class Meta:+ indexes = (GinIndex(fields=["alias"]),)+def __str__(self):"""Return a more human-readable representationdiff --git a/wger/nutrition/api/views.py b/wger/nutrition/api/views.pyindex a84471f22..976597fd1 100644--- a/wger/nutrition/api/views.py+++ b/wger/nutrition/api/views.py@@ -20,7 +20,7 @@ import logging# Djangofrom django.conf import settings-from django.contrib.postgres.search import SearchVector, SearchQuery, SearchRank+from django.contrib.postgres.search import TrigramSimilarityfrom django.utils.decorators import method_decoratorfrom django.views.decorators.cache import cache_page@@ -224,11 +224,11 @@ def search(request):# Postgres uses a full-text searchif is_postgres_db():- vector = SearchVector('search_column')- search_query = SearchQuery(term, search_type="websearch")- query = query.annotate(rank=SearchRank(vector, search_query)) \- .filter(search_column=search_query) \- .order_by('-rank', 'name')+ query = (+ query.annotate(similarity=TrigramSimilarity("name", term))+ .filter(similarity__gt=0.15)+ .order_by("-similarity", "name")+ )else:query = query.filter(name__icontains=term)diff --git a/wger/nutrition/migrations/0020_full_text_search.py b/wger/nutrition/migrations/0020_full_text_search.pyindex c377e3fb4..a4f6c20f0 100644--- a/wger/nutrition/migrations/0020_full_text_search.py+++ b/wger/nutrition/migrations/0020_full_text_search.py@@ -5,85 +5,16 @@ import django.contrib.postgres.searchfrom django.db import migrations, models-def forwards(apps, schema_editor):- if not schema_editor.connection.vendor.startswith('postgres'):- return-- schema_editor.execute('''- CREATE OR REPLACE FUNCTION update_ingredient_index() RETURNS trigger AS $$- DECLARE- known_langs TEXT[];- BEGIN- SELECT ARRAY(SELECT cfgname FROM pg_ts_config) INTO known_langs;-- -- Map the ingredient language to the list of known languages- IF NEW.language_id IS NOT NULL THEN- SELECT lower(full_name_en) INTO NEW.index_language- FROM core_language- WHERE id = NEW.language_id;-- -- If the language is not known values, just use English- IF NEW.index_language IS NULL OR NEW.index_language NOT IN (SELECT unnest(known_langs)) THEN- NEW.index_language = 'english';- END IF;- ELSE- -- If language_id is NULL, just set to English- NEW.index_language = 'english';- END IF;-- NEW.search_column :=- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(new.name,'')), 'A') ||- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(new.brand,'')), 'B') ||- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(new.common_name,'')), 'C');-- RETURN NEW;- END- $$ LANGUAGE plpgsql;-- -- Create the trigger- CREATE OR REPLACE TRIGGER ingredient_search_column_trigger- BEFORE INSERT OR UPDATE- ON nutrition_ingredient- FOR EACH ROW EXECUTE FUNCTION- update_ingredient_index();-- -- Force an update of the search index- UPDATE nutrition_ingredient SET name = name WHERE true;- ''')---def backwards(apps, schema_editor):- if not schema_editor.connection.vendor.startswith('postgres'):- return-- schema_editor.execute('''- DROP TRIGGER IF EXISTS ingredient_search_column_trigger ON nutrition_ingredient;- DROP FUNCTION IF EXISTS update_ingredient_index();- ''')--class Migration(migrations.Migration):dependencies = [- ('nutrition', '0019_alter_image_license_author_and_more'),+ ("nutrition", "0019_alter_image_license_author_and_more"),]operations = [- migrations.AddField(- model_name='ingredient',- name='index_language',- field=models.CharField(default='english', editable=False, max_length=30),- ),- migrations.AddField(- model_name='ingredient',- name='search_column',- field=django.contrib.postgres.search.SearchVectorField(editable=False, null=True),- ),migrations.AddIndex(- model_name='ingredient',+ model_name="ingredient",index=django.contrib.postgres.indexes.GinIndex(- fields=['search_column'],- name='nutrition_i_search__f274b7_gin'+ fields=["name"], name="nutrition_i_search__f274b7_gin"),),- migrations.RunPython(forwards, backwards)]diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.pyindex fc65dd77b..0c505f5f2 100644--- a/wger/nutrition/models/ingredient.py+++ b/wger/nutrition/models/ingredient.py@@ -239,25 +239,12 @@ class Ingredient(AbstractSubmissionModel, AbstractLicenseModel, models.Model):blank=True,)- index_language = models.CharField(- default='english',- editable=False,- max_length=30,- )- """The language used for the full text search"""-- search_column = SearchVectorField(- null=True,- editable=False,- )- """Column used for full text search"""-# Metaclass to set some other propertiesclass Meta:ordering = ["name",]- indexes = (GinIndex(fields=["search_column"]),)+ indexes = (GinIndex(fields=["name"]),)## Django methods