Skip to content

armgit / wger

Public Imported from GitHub
0

Self hosted FLOSS fitness/workout, nutrition and weight tracker

Add ingredient shadow table for ingredients

Roland Geider77291bdb649b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2d567805d..d8dbc2400 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -59,16 +59,16 @@ Also, we now support two-factor authentication like security codes or passkeys.
* If you are using docker, make sure to pull the latest changes from the
[docker repo](https://github.com/wger-project/docker) as there is a new service
- ("powersync") and configs (e.g. for nginx/Caddy).
-
-* `SIGNING_KEY` is replaced by `JWT_PRIVATE_KEY` and `JWT_PUBLIC_KEY`.
- Run `docker compose exec web ./manage.py generate-jwt-keys` to generate a new pair.
+ ("powersync") and configs (e.g. for nginx/Caddy) and settings (prod.env)
* The new powersync service needs its own dedicated user and schema inside the
Postgres database to store its sync state, run
`docker compose exec web ./manage.py setup-powersync-storage` to apply.
If you want to change the defaults, this can be configured in `PS_STORAGE_PG_URI`.
+* `SIGNING_KEY` is replaced by `JWT_PRIVATE_KEY` and `JWT_PUBLIC_KEY`.
+ Run `docker compose exec web ./manage.py generate-jwt-keys` to generate a new pair.
+
* Make sure you have set `SITE_URL` with your server's URL
* Some unused thumbnail sizes have been deleted, run `./manage.py prune-thumbnails`
diff --git a/wger/nutrition/migrations/0037_powersync_synced_ingredient_tables.py b/wger/nutrition/migrations/0037_powersync_synced_ingredient_tables.py
new file mode 100644
index 000000000..d50620089
--- /dev/null
+++ b/wger/nutrition/migrations/0037_powersync_synced_ingredient_tables.py
@@ -0,0 +1,150 @@
+from django.db import migrations
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+
+from wger.utils.db import postgres_only
+
+"""
+Adds a slim shadow copy of nutrition_ingredient.
+
+PowerSync materialises one bucket per *source row* a data query matches, so
+syncing ``nutrition_ingredient`` directly creates one bucket per ingredient
+(~3M) even though only the ingredients actually referenced by some log/meal
+item are ever used. That bloats bucket storage and the initial-replication
+footprint (the WAL grows ~25GB while the snapshot runs).
+
+To avoid this the sync rules read ingredients from ``nutrition_synced_ingredient``,
+a shadow table that holds only the *used* rows, kept in sync by DB triggers (this
+makes sure that the data is mirrored regardless of how the write happens: ORM,
+bulk_create, COPY, raw SQL).
+
+Other tables such as ``nutrition_image`` and ``nutrition_ingredientweightunit``
+are synced directly, since at least currently it doesn't seem necessary to add
+any shadow tables of their own.
+
+
+Keeping the shadow table in lockstep when the ingredient schema changes
+-----------------------------------------------------------------------
+``nutrition_synced_ingredient`` is a physical copy and the triggers/functions
+copy with ``SELECT *``, so any structure change to ``nutrition_ingredient`` must
+be mirrored here, in the SAME migration that alters the source model:
+
+1. Mirror the change on ``nutrition_synced_ingredient``, e.g.
+ ``ALTER TABLE nutrition_synced_ingredient ADD COLUMN <col> <type>;``
+ (likewise for renames, drops and type changes, the columns must be kept
+ identical in name, type and order).
+
+2. Backfill the new column into existing shadow rows:
+ ``UPDATE nutrition_synced_ingredient s SET <col> = i.<col>``
+ ``FROM nutrition_ingredient i WHERE s.id = i.id;``
+ This is required: Django's AddField-with-default is a metadata-only change on
+ Postgres >= 11 and does NOT fire the AFTER UPDATE trigger, so existing shadow
+ rows would otherwise keep NULL / the default.
+
+3. The functions and triggers themselves need no change, as long as the shadow
+ stays structurally identical. If it drifts, the ``SELECT *`` copy fails and
+ log/meal-item INSERTs error out loudly -- that is the intended guard.
+"""
+
+
+
+SQL = [
+ # Shadow table: structure cloned incl. PK (= replica identity for logical
+ # replication). EXCLUDING IDENTITY so we can insert the explicit ids copied
+ # from the source rows.
+ """
+ CREATE TABLE IF NOT EXISTS nutrition_synced_ingredient(
+ LIKE nutrition_ingredient INCLUDING ALL EXCLUDING IDENTITY
+ );
+ """,
+
+ # Copy an ingredient into the shadow set
+ """
+ CREATE OR REPLACE FUNCTION powersync_ensure_ingredient_synced(ing_id bigint)
+ RETURNS void LANGUAGE sql AS $$
+ INSERT INTO nutrition_synced_ingredient
+ SELECT * FROM nutrition_ingredient WHERE id = ing_id
+ ON CONFLICT (id) DO NOTHING;
+ $$;
+ """,
+
+ # On every log and meal item, pull its ingredient into the shadow set.
+ """
+ CREATE OR REPLACE FUNCTION powersync_sync_ingredient_from_item()
+ RETURNS trigger LANGUAGE plpgsql AS $$
+ BEGIN
+ PERFORM powersync_ensure_ingredient_synced(NEW.ingredient_id);
+ RETURN NEW;
+ END $$;
+ """,
+ """
+ CREATE TRIGGER powersync_sync_ingredient
+ AFTER INSERT OR UPDATE OF ingredient_id ON nutrition_logitem
+ FOR EACH ROW EXECUTE FUNCTION powersync_sync_ingredient_from_item();
+ """,
+ """
+ CREATE TRIGGER powersync_sync_ingredient
+ AFTER INSERT OR UPDATE OF ingredient_id ON nutrition_mealitem
+ FOR EACH ROW EXECUTE FUNCTION powersync_sync_ingredient_from_item();
+ """,
+
+ # Keep already-synced ingredient rows fresh when the source changes.
+ """
+ CREATE OR REPLACE FUNCTION powersync_propagate_ingredient()
+ RETURNS trigger LANGUAGE plpgsql AS $$
+ BEGIN
+ IF EXISTS (SELECT 1 FROM nutrition_synced_ingredient WHERE id = NEW.id) THEN
+ DELETE FROM nutrition_synced_ingredient WHERE id = NEW.id;
+ INSERT INTO nutrition_synced_ingredient
+ SELECT * FROM nutrition_ingredient WHERE id = NEW.id;
+ END IF;
+ RETURN NEW;
+ END $$;
+ """,
+ """
+ CREATE TRIGGER powersync_propagate
+ AFTER UPDATE ON nutrition_ingredient
+ FOR EACH ROW EXECUTE FUNCTION powersync_propagate_ingredient();
+ """,
+
+ # Backfill everything already referenced by a log or meal item.
+ """
+ SELECT powersync_ensure_ingredient_synced(id) FROM (
+ SELECT DISTINCT ingredient_id AS id FROM nutrition_logitem
+ UNION
+ SELECT DISTINCT ingredient_id FROM nutrition_mealitem
+ ) used;
+ """,
+]
+
+REVERSE_SQL = [
+ "DROP TRIGGER IF EXISTS powersync_propagate ON nutrition_ingredient;",
+ "DROP TRIGGER IF EXISTS powersync_sync_ingredient ON nutrition_mealitem;",
+ "DROP TRIGGER IF EXISTS powersync_sync_ingredient ON nutrition_logitem;",
+ "DROP FUNCTION IF EXISTS powersync_propagate_ingredient();",
+ "DROP FUNCTION IF EXISTS powersync_sync_ingredient_from_item();",
+ "DROP FUNCTION IF EXISTS powersync_ensure_ingredient_synced(bigint);",
+ "DROP TABLE IF EXISTS nutrition_synced_ingredient;",
+]
+
+
+@postgres_only
+def create_synced_tables(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor):
+ for statement in SQL:
+ schema_editor.execute(statement)
+
+
+@postgres_only
+def drop_synced_tables(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor):
+ for statement in REVERSE_SQL:
+ schema_editor.execute(statement)
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ('nutrition', '0036_alter_image_license_author_and_more'),
+ ]
+
+ operations = [
+ migrations.RunPython(create_synced_tables, reverse_code=drop_synced_tables),
+ ]
diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.py
index ecefa08a7..6e02f1655 100644
--- a/wger/nutrition/models/ingredient.py
+++ b/wger/nutrition/models/ingredient.py
@@ -62,6 +62,10 @@ logger = logging.getLogger(__name__)
class Ingredient(AbstractLicenseModel, models.Model):
"""
An ingredient, with some approximate nutrition values
+
+ IMPORTANT: when changing, adding or removing fields here, make sure to add
+ them to the ingredient shadow sync table. See the comment in the
+ 0037_powersync_synced_ingredient_tables migration for details.
"""
class Meta: