Node.js based forum software built for the modern web
Fix the saving of the statistics on PosgreSQL #14124 (#14129)
Michele Di Mariab8fd88fba955
diff --git a/src/analytics.js b/src/analytics.jsindex f64c651687..e8e60c5e68 100644--- a/src/analytics.js+++ b/src/analytics.js@@ -189,6 +189,12 @@ Analytics.writeData = async function () {incrByBulk.push(['analytics:pageviews:ap', total.apPageViews, today.getTime()]);incrByBulk.push(['analytics:pageviews:ap:month', total.apPageViews, month.getTime()]);total.apPageViews = 0;+ if (!metrics.includes('pageviews:ap')) {+ metrics.push('pageviews:ap');+ }+ if (!metrics.includes('pageviews:ap:month')) {+ metrics.push('pageviews:ap:month');+ }}if (total.uniquevisitors > 0) {diff --git a/src/database/postgres/helpers.js b/src/database/postgres/helpers.jsindex 85e0b63d07..8b92d3fe50 100644--- a/src/database/postgres/helpers.js+++ b/src/database/postgres/helpers.js@@ -27,31 +27,25 @@ DELETE FROM "legacy_object"AND "expireAt" <= CURRENT_TIMESTAMP`,});- await db.query({- name: 'ensureLegacyObjectType1',+ const res = await db.query({+ name: 'ensureLegacyObjectType_upsert',text: `INSERT INTO "legacy_object" ("_key", "type")VALUES ($1::TEXT, $2::TEXT::LEGACY_OBJECT_TYPE)- ON CONFLICT- DO NOTHING`,+ ON CONFLICT ("_key")+ DO UPDATE SET "type" = "legacy_object"."type"+ RETURNING "type"`,values: [key, type],});- const res = await db.query({- name: 'ensureLegacyObjectType2',- text: `-SELECT "type"- FROM "legacy_object_live"- WHERE "_key" = $1::TEXT`,- values: [key],- });-if (res.rows[0].type !== type) {throw new Error(`database: cannot insert ${JSON.stringify(key)} as ${type} because it already exists as ${res.rows[0].type}`);}};helpers.ensureLegacyObjectsType = async function (db, keys, type) {+ keys = [...new Set(keys)];+await db.query({name: 'ensureLegacyObjectTypeBefore',text: `@@ -60,38 +54,24 @@ DELETE FROM "legacy_object"AND "expireAt" <= CURRENT_TIMESTAMP`,});- await db.query({- name: 'ensureLegacyObjectsType1',+ const res = await db.query({+ name: 'ensureLegacyObjectsType_upsert',text: `INSERT INTO "legacy_object" ("_key", "type")SELECT k, $2::TEXT::LEGACY_OBJECT_TYPEFROM UNNEST($1::TEXT[]) k- ON CONFLICT- DO NOTHING`,+ ON CONFLICT ("_key")+ DO UPDATE SET "type" = "legacy_object"."type"+ RETURNING "_key", "type"`,values: [keys, type],});- const res = await db.query({- name: 'ensureLegacyObjectsType2',- text: `-SELECT "_key", "type"- FROM "legacy_object_live"- WHERE "_key" = ANY($1::TEXT[])`,- values: [keys],- });-const invalid = res.rows.filter(r => r.type !== type);if (invalid.length) {const parts = invalid.map(r => `${JSON.stringify(r._key)} is ${r.type}`);throw new Error(`database: cannot insert multiple objects as ${type} because they already exist: ${parts.join(', ')}`);}-- const missing = keys.filter(k => !res.rows.some(r => r._key === k));-- if (missing.length) {- throw new Error(`database: failed to insert keys for objects: ${JSON.stringify(missing)}`);- }};helpers.noop = function () {};diff --git a/src/database/postgres/sorted.js b/src/database/postgres/sorted.jsindex 351fe3e059..50581a6ed4 100644--- a/src/database/postgres/sorted.js+++ b/src/database/postgres/sorted.js@@ -551,16 +551,29 @@ RETURNING "score" s`,return [];}+ // Deduplicate by (key, value) pair, summing increments for duplicates+ const seen = new Map();+ const deduped = [];+ data.forEach(([key, increment, value]) => {+ value = helpers.valueToString(value);+ increment = parseFloat(increment);+ const mapKey = `${key}\0${value}`;+ if (seen.has(mapKey)) {+ deduped[seen.get(mapKey)][1] += increment;+ } else {+ seen.set(mapKey, deduped.length);+ deduped.push([key, increment, value]);+ }+ });+return await module.transaction(async (client) => {- await helpers.ensureLegacyObjectsType(client, data.map(item => item[0]), 'zset');+ await helpers.ensureLegacyObjectsType(client, deduped.map(item => item[0]), 'zset');const values = [];const queryParams = [];let paramIndex = 1;- data.forEach(([key, increment, value]) => {- value = helpers.valueToString(value);- increment = parseFloat(increment);+ deduped.forEach(([key, increment, value]) => {values.push(key, value, increment);queryParams.push(`($${paramIndex}::TEXT, $${paramIndex + 1}::TEXT, $${paramIndex + 2}::NUMERIC)`);paramIndex += 3;diff --git a/src/database/postgres/sorted/add.js b/src/database/postgres/sorted/add.jsindex 6f87416089..3db65cfde9 100644--- a/src/database/postgres/sorted/add.js+++ b/src/database/postgres/sorted/add.js@@ -114,8 +114,10 @@ INSERT INTO "legacy_zset" ("_key", "value", "score")}keys.push(item[0]);scores.push(item[1]);- values.push(item[2]);+ values.push(helpers.valueToString(item[2]));});+ const compositeKeys = keys.map((k, i) => `${k}\0${values[i]}`);+ helpers.removeDuplicateValues(compositeKeys, keys, values, scores);await module.transaction(async (client) => {await helpers.ensureLegacyObjectsType(client, keys, 'zset');await client.query({