Skip to content

Node.js based forum software built for the modern web

Fix the saving of the statistics on PosgreSQL #14124 (#14129)

Michele Di Mariab8fd88fba955
diff --git a/src/analytics.js b/src/analytics.js
index f64c651687..e8e60c5e68 100644
--- a/src/analytics.js
+++ b/src/analytics.js
@@ -189,6 +189,12 @@ Analytics.writeData = async function () {
incrByBulk.push(['analytics:pageviews:ap', total.apPageViews, today.getTime()]);
incrByBulk.push(['analytics:pageviews:ap:month', total.apPageViews, month.getTime()]);
total.apPageViews = 0;
+ if (!metrics.includes('pageviews:ap')) {
+ metrics.push('pageviews:ap');
+ }
+ if (!metrics.includes('pageviews:ap:month')) {
+ metrics.push('pageviews:ap:month');
+ }
}
if (total.uniquevisitors > 0) {
diff --git a/src/database/postgres/helpers.js b/src/database/postgres/helpers.js
index 85e0b63d07..8b92d3fe50 100644
--- a/src/database/postgres/helpers.js
+++ b/src/database/postgres/helpers.js
@@ -27,31 +27,25 @@ DELETE FROM "legacy_object"
AND "expireAt" <= CURRENT_TIMESTAMP`,
});
- await db.query({
- name: 'ensureLegacyObjectType1',
+ const res = await db.query({
+ name: 'ensureLegacyObjectType_upsert',
text: `
INSERT INTO "legacy_object" ("_key", "type")
VALUES ($1::TEXT, $2::TEXT::LEGACY_OBJECT_TYPE)
- ON CONFLICT
- DO NOTHING`,
+ ON CONFLICT ("_key")
+ DO UPDATE SET "type" = "legacy_object"."type"
+ RETURNING "type"`,
values: [key, type],
});
- const res = await db.query({
- name: 'ensureLegacyObjectType2',
- text: `
-SELECT "type"
- FROM "legacy_object_live"
- WHERE "_key" = $1::TEXT`,
- values: [key],
- });
-
if (res.rows[0].type !== type) {
throw new Error(`database: cannot insert ${JSON.stringify(key)} as ${type} because it already exists as ${res.rows[0].type}`);
}
};
helpers.ensureLegacyObjectsType = async function (db, keys, type) {
+ keys = [...new Set(keys)];
+
await db.query({
name: 'ensureLegacyObjectTypeBefore',
text: `
@@ -60,38 +54,24 @@ DELETE FROM "legacy_object"
AND "expireAt" <= CURRENT_TIMESTAMP`,
});
- await db.query({
- name: 'ensureLegacyObjectsType1',
+ const res = await db.query({
+ name: 'ensureLegacyObjectsType_upsert',
text: `
INSERT INTO "legacy_object" ("_key", "type")
SELECT k, $2::TEXT::LEGACY_OBJECT_TYPE
FROM UNNEST($1::TEXT[]) k
- ON CONFLICT
- DO NOTHING`,
+ ON CONFLICT ("_key")
+ DO UPDATE SET "type" = "legacy_object"."type"
+ RETURNING "_key", "type"`,
values: [keys, type],
});
- const res = await db.query({
- name: 'ensureLegacyObjectsType2',
- text: `
-SELECT "_key", "type"
- FROM "legacy_object_live"
- WHERE "_key" = ANY($1::TEXT[])`,
- values: [keys],
- });
-
const invalid = res.rows.filter(r => r.type !== type);
if (invalid.length) {
const parts = invalid.map(r => `${JSON.stringify(r._key)} is ${r.type}`);
throw new Error(`database: cannot insert multiple objects as ${type} because they already exist: ${parts.join(', ')}`);
}
-
- const missing = keys.filter(k => !res.rows.some(r => r._key === k));
-
- if (missing.length) {
- throw new Error(`database: failed to insert keys for objects: ${JSON.stringify(missing)}`);
- }
};
helpers.noop = function () {};
diff --git a/src/database/postgres/sorted.js b/src/database/postgres/sorted.js
index 351fe3e059..50581a6ed4 100644
--- a/src/database/postgres/sorted.js
+++ b/src/database/postgres/sorted.js
@@ -551,16 +551,29 @@ RETURNING "score" s`,
return [];
}
+ // Deduplicate by (key, value) pair, summing increments for duplicates
+ const seen = new Map();
+ const deduped = [];
+ data.forEach(([key, increment, value]) => {
+ value = helpers.valueToString(value);
+ increment = parseFloat(increment);
+ const mapKey = `${key}\0${value}`;
+ if (seen.has(mapKey)) {
+ deduped[seen.get(mapKey)][1] += increment;
+ } else {
+ seen.set(mapKey, deduped.length);
+ deduped.push([key, increment, value]);
+ }
+ });
+
return await module.transaction(async (client) => {
- await helpers.ensureLegacyObjectsType(client, data.map(item => item[0]), 'zset');
+ await helpers.ensureLegacyObjectsType(client, deduped.map(item => item[0]), 'zset');
const values = [];
const queryParams = [];
let paramIndex = 1;
- data.forEach(([key, increment, value]) => {
- value = helpers.valueToString(value);
- increment = parseFloat(increment);
+ deduped.forEach(([key, increment, value]) => {
values.push(key, value, increment);
queryParams.push(`($${paramIndex}::TEXT, $${paramIndex + 1}::TEXT, $${paramIndex + 2}::NUMERIC)`);
paramIndex += 3;
diff --git a/src/database/postgres/sorted/add.js b/src/database/postgres/sorted/add.js
index 6f87416089..3db65cfde9 100644
--- a/src/database/postgres/sorted/add.js
+++ b/src/database/postgres/sorted/add.js
@@ -114,8 +114,10 @@ INSERT INTO "legacy_zset" ("_key", "value", "score")
}
keys.push(item[0]);
scores.push(item[1]);
- values.push(item[2]);
+ values.push(helpers.valueToString(item[2]));
});
+ const compositeKeys = keys.map((k, i) => `${k}\0${values[i]}`);
+ helpers.removeDuplicateValues(compositeKeys, keys, values, scores);
await module.transaction(async (client) => {
await helpers.ensureLegacyObjectsType(client, keys, 'zset');
await client.query({