Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Use a simple trigram search

Roland Geiderc590299e56b2
diff --git a/wger/exercises/api/views.py b/wger/exercises/api/views.py
index 3ed69fe64..16b1eaf02 100644
--- a/wger/exercises/api/views.py
+++ b/wger/exercises/api/views.py
@@ -17,18 +17,17 @@
import logging
from uuid import UUID
+# Third Party
+import bleach
+from actstream import action as actstream_action
+from bleach.css_sanitizer import CSSSanitizer
# Django
from django.conf import settings
-from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector
+from django.contrib.postgres.search import TrigramSimilarity
from django.db.models import Q
from django.utils.decorators import method_decorator
from django.utils.translation import gettext as _
from django.views.decorators.cache import cache_page
-
-# Third Party
-import bleach
-from actstream import action as actstream_action
-from bleach.css_sanitizer import CSSSanitizer
from drf_spectacular.types import OpenApiTypes
from drf_spectacular.utils import (
OpenApiParameter,
@@ -100,6 +99,7 @@ class ExerciseBaseViewSet(ModelViewSet):
For a read-only endpoint with all the information of an exercise, see /api/v2/exercisebaseinfo/
"""
+
queryset = ExerciseBase.translations.all()
serializer_class = ExerciseBaseSerializer
permission_classes = (CanContributeExercises,)
@@ -149,6 +149,7 @@ class ExerciseTranslationViewSet(ModelViewSet):
"""
API endpoint for editing or adding exercise translation objects.
"""
+
queryset = Exercise.objects.all()
permission_classes = (CanContributeExercises,)
serializer_class = ExerciseTranslationSerializer
@@ -350,16 +351,14 @@ def search(request):
# Postgres uses a full-text search
if is_postgres_db():
- vector = SearchVector('search_column')
- search_query = SearchQuery(q, search_type="websearch")
- query = query.annotate(rank=SearchRank(vector, search_query)) \
- .filter(search_column=search_query) \
- .order_by('-rank', 'name')
+ query = (
+ query.annotate(similarity=TrigramSimilarity("name", q))
+ .filter(Q(similarity__gt=0.15) | Q(alias__alias__icontains=q))
+ .order_by("-similarity", "name")
+ )
else:
query = query.filter(Q(name__icontains=q) | Q(alias__alias__icontains=q))
- print(query.query)
-
for translation in query:
image = None
thumbnail = None
@@ -385,7 +384,7 @@ def search(request):
}
}
results.append(result_json)
- response['suggestions'] = results
+ response["suggestions"] = results
return Response(response)
diff --git a/wger/exercises/migrations/0029_full_text_search.py b/wger/exercises/migrations/0029_full_text_search.py
index 174fbf422..3ba780cf9 100644
--- a/wger/exercises/migrations/0029_full_text_search.py
+++ b/wger/exercises/migrations/0029_full_text_search.py
@@ -1,99 +1,29 @@
# Generated by Django 4.2.6 on 2024-01-18 16:55
import django.contrib.postgres.search
+from django.contrib.postgres.operations import TrigramExtension, BtreeGinExtension
from django.db import migrations, models
-def forwards(apps, schema_editor):
- if not schema_editor.connection.vendor.startswith('postgres'):
- return
-
- schema_editor.execute('''
- CREATE OR REPLACE FUNCTION update_exercise_fulltext_index() RETURNS trigger AS $$
- DECLARE
- aliases TEXT;
- known_langs TEXT[];
- BEGIN
- SELECT ARRAY(SELECT cfgname FROM pg_ts_config) INTO known_langs;
-
- IF NEW.language_id IS NOT NULL THEN
- SELECT lower(full_name_en)
- INTO NEW.index_language
- FROM core_language
- WHERE id = NEW.language_id;
-
- -- If the language is not known, just use English
- IF NEW.index_language IS NULL OR NEW.index_language NOT IN (SELECT unnest(known_langs)) THEN
- NEW.index_language = 'english';
- END IF;
- ELSE
- -- If language_id is NULL, just set to English
- NEW.index_language = 'english';
- END IF;
-
- SELECT COALESCE(string_agg(a.alias, ' '), '')
- INTO aliases
- FROM exercises_exercise AS t
- LEFT JOIN exercises_alias AS a
- ON t.id = a.exercise_id
- WHERE a.exercise_id = NEW.id;
-
- NEW.search_column :=
- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(new.name, '')), 'A') ||
- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(aliases, '')), 'A') ||
- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(new.description, '')), 'D');
-
- RETURN NEW;
- END
- $$ LANGUAGE plpgsql;
-
- -- Create the trigger
- CREATE OR REPLACE TRIGGER exercise_search_column_trigger
- BEFORE INSERT OR UPDATE
- ON exercises_exercise
- FOR EACH ROW EXECUTE FUNCTION
- update_exercise_fulltext_index();
-
- -- Force an update of the search index
- UPDATE exercises_exercise SET name = name WHERE true;
- ''')
-
-
-def backwards(apps, schema_editor):
- if not schema_editor.connection.vendor.startswith('postgres'):
- return
-
- schema_editor.execute('''
- DROP TRIGGER IF EXISTS exercise_search_column_trigger ON exercises_exercise;
- DROP FUNCTION IF EXISTS update_exercise_fulltext_index();
- ''')
-
-
class Migration(migrations.Migration):
dependencies = [
('exercises', '0028_add_uuid_alias_and_comments'),
]
operations = [
- migrations.AddField(
- model_name='exercise',
- name='index_language',
- field=models.CharField(default='english', editable=False, max_length=30),
- ),
- migrations.AddField(
- model_name='exercise',
- name='search_column',
- field=django.contrib.postgres.search.SearchVectorField(editable=False, null=True),
- ),
- migrations.AddField(
- model_name='historicalexercise',
- name='index_language',
- field=models.CharField(default='english', editable=False, max_length=30),
+ BtreeGinExtension(),
+ TrigramExtension(),
+ migrations.AddIndex(
+ model_name="exercise",
+ index=django.contrib.postgres.indexes.GinIndex(
+ fields=["name"],
+ name="exercises_e_name_ac11f4_gin",
+ ),
),
- migrations.AddField(
- model_name='historicalexercise',
- name='search_column',
- field=django.contrib.postgres.search.SearchVectorField(editable=False, null=True),
+ migrations.AddIndex(
+ model_name="alias",
+ index=django.contrib.postgres.indexes.GinIndex(
+ fields=["alias"], name="exercises_a_alias_227e38_gin"
+ ),
),
- migrations.RunPython(forwards, backwards)
]
diff --git a/wger/exercises/models/exercise.py b/wger/exercises/models/exercise.py
index d769d9534..0ca2a8e9b 100644
--- a/wger/exercises/models/exercise.py
+++ b/wger/exercises/models/exercise.py
@@ -18,7 +18,7 @@
import uuid
# Django
-from django.contrib.postgres.search import SearchVectorField
+from django.contrib.postgres.indexes import GinIndex
from django.core.validators import MinLengthValidator
from django.db import models
from django.urls import reverse
@@ -96,19 +96,6 @@ class Exercise(AbstractLicenseModel, AbstractHistoryMixin, models.Model):
)
""" Refers to the base exercise with non translated information """
- search_column = SearchVectorField(
- null=True,
- editable=False,
- )
- """Column used for full text search"""
-
- index_language = models.CharField(
- default='english',
- editable=False,
- max_length=30,
- )
- """The language used for the full text search"""
-
history = HistoricalRecords()
"""Edit history"""
@@ -120,6 +107,7 @@ class Exercise(AbstractLicenseModel, AbstractHistoryMixin, models.Model):
ordering = [
"name",
]
+ indexes = (GinIndex(fields=["name"]),)
def get_absolute_url(self):
"""
diff --git a/wger/exercises/models/exercise_alias.py b/wger/exercises/models/exercise_alias.py
index 9ff823a53..e470222a4 100644
--- a/wger/exercises/models/exercise_alias.py
+++ b/wger/exercises/models/exercise_alias.py
@@ -18,6 +18,7 @@
import uuid
# Django
+from django.contrib.postgres.indexes import GinIndex
from django.db import models
from django.utils.translation import gettext_lazy as _
@@ -38,6 +39,7 @@ class Alias(models.Model):
"""
Model for an exercise (name)alias
"""
+
uuid = models.UUIDField(
default=uuid.uuid4,
editable=False,
@@ -60,6 +62,9 @@ class Alias(models.Model):
history = HistoricalRecords()
"""Edit history"""
+ class Meta:
+ indexes = (GinIndex(fields=["alias"]),)
+
def __str__(self):
"""
Return a more human-readable representation
diff --git a/wger/nutrition/api/views.py b/wger/nutrition/api/views.py
index a84471f22..976597fd1 100644
--- a/wger/nutrition/api/views.py
+++ b/wger/nutrition/api/views.py
@@ -20,7 +20,7 @@ import logging
# Django
from django.conf import settings
-from django.contrib.postgres.search import SearchVector, SearchQuery, SearchRank
+from django.contrib.postgres.search import TrigramSimilarity
from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_page
@@ -224,11 +224,11 @@ def search(request):
# Postgres uses a full-text search
if is_postgres_db():
- vector = SearchVector('search_column')
- search_query = SearchQuery(term, search_type="websearch")
- query = query.annotate(rank=SearchRank(vector, search_query)) \
- .filter(search_column=search_query) \
- .order_by('-rank', 'name')
+ query = (
+ query.annotate(similarity=TrigramSimilarity("name", term))
+ .filter(similarity__gt=0.15)
+ .order_by("-similarity", "name")
+ )
else:
query = query.filter(name__icontains=term)
diff --git a/wger/nutrition/migrations/0020_full_text_search.py b/wger/nutrition/migrations/0020_full_text_search.py
index c377e3fb4..a4f6c20f0 100644
--- a/wger/nutrition/migrations/0020_full_text_search.py
+++ b/wger/nutrition/migrations/0020_full_text_search.py
@@ -5,85 +5,16 @@ import django.contrib.postgres.search
from django.db import migrations, models
-def forwards(apps, schema_editor):
- if not schema_editor.connection.vendor.startswith('postgres'):
- return
-
- schema_editor.execute('''
- CREATE OR REPLACE FUNCTION update_ingredient_index() RETURNS trigger AS $$
- DECLARE
- known_langs TEXT[];
- BEGIN
- SELECT ARRAY(SELECT cfgname FROM pg_ts_config) INTO known_langs;
-
- -- Map the ingredient language to the list of known languages
- IF NEW.language_id IS NOT NULL THEN
- SELECT lower(full_name_en) INTO NEW.index_language
- FROM core_language
- WHERE id = NEW.language_id;
-
- -- If the language is not known values, just use English
- IF NEW.index_language IS NULL OR NEW.index_language NOT IN (SELECT unnest(known_langs)) THEN
- NEW.index_language = 'english';
- END IF;
- ELSE
- -- If language_id is NULL, just set to English
- NEW.index_language = 'english';
- END IF;
-
- NEW.search_column :=
- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(new.name,'')), 'A') ||
- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(new.brand,'')), 'B') ||
- setweight(to_tsvector(NEW.index_language::regconfig, coalesce(new.common_name,'')), 'C');
-
- RETURN NEW;
- END
- $$ LANGUAGE plpgsql;
-
- -- Create the trigger
- CREATE OR REPLACE TRIGGER ingredient_search_column_trigger
- BEFORE INSERT OR UPDATE
- ON nutrition_ingredient
- FOR EACH ROW EXECUTE FUNCTION
- update_ingredient_index();
-
- -- Force an update of the search index
- UPDATE nutrition_ingredient SET name = name WHERE true;
- ''')
-
-
-def backwards(apps, schema_editor):
- if not schema_editor.connection.vendor.startswith('postgres'):
- return
-
- schema_editor.execute('''
- DROP TRIGGER IF EXISTS ingredient_search_column_trigger ON nutrition_ingredient;
- DROP FUNCTION IF EXISTS update_ingredient_index();
- ''')
-
-
class Migration(migrations.Migration):
dependencies = [
- ('nutrition', '0019_alter_image_license_author_and_more'),
+ ("nutrition", "0019_alter_image_license_author_and_more"),
]
operations = [
- migrations.AddField(
- model_name='ingredient',
- name='index_language',
- field=models.CharField(default='english', editable=False, max_length=30),
- ),
- migrations.AddField(
- model_name='ingredient',
- name='search_column',
- field=django.contrib.postgres.search.SearchVectorField(editable=False, null=True),
- ),
migrations.AddIndex(
- model_name='ingredient',
+ model_name="ingredient",
index=django.contrib.postgres.indexes.GinIndex(
- fields=['search_column'],
- name='nutrition_i_search__f274b7_gin'
+ fields=["name"], name="nutrition_i_search__f274b7_gin"
),
),
- migrations.RunPython(forwards, backwards)
]
diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.py
index fc65dd77b..0c505f5f2 100644
--- a/wger/nutrition/models/ingredient.py
+++ b/wger/nutrition/models/ingredient.py
@@ -239,25 +239,12 @@ class Ingredient(AbstractSubmissionModel, AbstractLicenseModel, models.Model):
blank=True,
)
- index_language = models.CharField(
- default='english',
- editable=False,
- max_length=30,
- )
- """The language used for the full text search"""
-
- search_column = SearchVectorField(
- null=True,
- editable=False,
- )
- """Column used for full text search"""
-
# Metaclass to set some other properties
class Meta:
ordering = [
"name",
]
- indexes = (GinIndex(fields=["search_column"]),)
+ indexes = (GinIndex(fields=["name"]),)
#
# Django methods