Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Make test stats upload more robust (#77258)

Fraser384f6fb9b76d
diff --git a/.github/scripts/csv-to-mb.js b/.github/scripts/csv-to-mb.js
index 5f5de281371..a9f024db823 100644
--- a/.github/scripts/csv-to-mb.js
+++ b/.github/scripts/csv-to-mb.js
@@ -30,7 +30,7 @@ class UploadError extends Error {
}
}
-async function attemptUpload(url, jsonData) {
+async function attemptUpload(url, jsonData, timeoutMs) {
let response;
try {
response = await fetch(url, {
@@ -39,7 +39,11 @@ async function attemptUpload(url, jsonData) {
"x-api-key": process.env.API_KEY
},
// Rebuilt per attempt: a consumed request body can't be reused.
- body: jsonToCsvFormData(jsonData)
+ body: jsonToCsvFormData(jsonData),
+ // Without this the request can hang indefinitely on a stalled server,
+ // eating the whole job timeout. An abort surfaces below as a retryable
+ // network error, so it retries with backoff instead.
+ signal: AbortSignal.timeout(timeoutMs)
});
} catch (networkError) {
throw new UploadError(networkError.message, { retryable: true });
@@ -61,16 +65,17 @@ const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
/**
* Uploads jsonData as CSV, retrying transient API errors with exponential
* backoff. `retries` is the number of extra attempts after the first (so the
- * default 2 means up to 3 tries). Throws the last error once attempts are
- * exhausted or the error is non-retryable.
+ * default 2 means up to 3 tries). `timeoutMs` is a per-attempt total deadline
+ * (default 60s) that aborts a stalled request. Throws the last error once
+ * attempts are exhausted or the error is non-retryable.
*/
-async function uploadCsvToMb({ baseUrl, tableId, jsonData, mode = 'append', retries = 2, retryDelayMs = 1000 }) {
+async function uploadCsvToMb({ baseUrl, tableId, jsonData, mode = 'append', retries = 2, retryDelayMs = 1000, timeoutMs = 60_000 }) {
const operation = mode === 'replace' ? 'replace-csv' : 'append-csv';
const url = `${baseUrl}/api/table/${tableId}/${operation}`;
for (let attempt = 0; ; attempt++) {
try {
- await attemptUpload(url, jsonData);
+ await attemptUpload(url, jsonData, timeoutMs);
return { success: true };
} catch (error) {
if (!error.retryable || attempt >= retries) {
diff --git a/.github/scripts/upload-affected-tests-stats.js b/.github/scripts/upload-affected-tests-stats.js
index 4c9731cec31..7f4f98ced4a 100644
--- a/.github/scripts/upload-affected-tests-stats.js
+++ b/.github/scripts/upload-affected-tests-stats.js
@@ -7,48 +7,58 @@ const { uploadCsvToMb } = require("./csv-to-mb.js");
// "FE Affected Tests" uploaded table on stats.metabase.com.
const TABLE_ID = 81818;
-const s = JSON.parse(process.env.STATS_JSON);
-console.log("stats", s);
+async function main() {
+ // create-test-plan doesn't always produce stats (e.g. on release branches
+ // running an older planner), leaving STATS_JSON empty. Nothing to upload.
+ const raw = (process.env.STATS_JSON || "").trim();
+ if (!raw || raw === "null") {
+ console.log("No STATS_JSON to upload; skipping.");
+ return;
+ }
-const row = {
- Date: new Date().toISOString(),
- // "pr_update" or "merge_to_master". ("Trigger" is a reserved SQL word.)
- "Triggered By": process.env.TRIGGERED_BY,
- PR: Number(process.env.PR_NUMBER),
- "Head SHA": process.env.HEAD_SHA,
- "Base SHA": process.env.BASE_SHA,
- "FE Files Changed": s.fe_files_changed,
- "FE Files Total": s.fe_files_total,
- "BE Files Changed": s.be_files_changed,
- "BE Files Total": s.be_files_total,
- "Unit Infra Touched": s.unit_infra_touched,
- "Loki Infra Touched": s.loki_infra_touched,
- "Shared Sources Touched": s.shared_sources_touched,
- "FE Modules Total": s.fe_modules_total,
- "FE Modules Changed": s.fe_modules_changed,
- "FE Modules Affected (Rules)": s.fe_modules_affected_rules,
- "FE Modules Affected (Usage)": s.fe_modules_affected_usage,
- "Unit Specs All": s.unit_specs_all,
- "Unit Specs To Run (Rules)": s.unit_specs_to_run_rules,
- "Unit Specs To Run (Usage)": s.unit_specs_to_run_usage,
- "Loki Stories All": s.loki_stories_all,
- "Loki Stories To Run (Rules)": s.loki_stories_to_run_rules,
- "Loki Stories To Run (Usage)": s.loki_stories_to_run_usage,
- "E2E Specs All": s.e2e_specs_all,
- "E2E Specs To Run (Rules)": s.e2e_specs_to_run_rules,
- "E2E Specs To Run (Usage)": s.e2e_specs_to_run_usage,
-};
+ const s = JSON.parse(raw);
+ console.log("stats", s);
-uploadCsvToMb({
- baseUrl: "https://stats.metabase.com",
- tableId: TABLE_ID,
- jsonData: [row],
- mode: "append",
-})
- .then(() => {
- console.log("Data uploaded successfully");
- })
- .catch((error) => {
- console.error("Error uploading data:", error);
- process.exit(1);
+ const row = {
+ Date: new Date().toISOString(),
+ // "pr_update" or "merge_to_master". ("Trigger" is a reserved SQL word.)
+ "Triggered By": process.env.TRIGGERED_BY,
+ PR: Number(process.env.PR_NUMBER),
+ "Head SHA": process.env.HEAD_SHA,
+ "Base SHA": process.env.BASE_SHA,
+ "FE Files Changed": s.fe_files_changed,
+ "FE Files Total": s.fe_files_total,
+ "BE Files Changed": s.be_files_changed,
+ "BE Files Total": s.be_files_total,
+ "Unit Infra Touched": s.unit_infra_touched,
+ "Loki Infra Touched": s.loki_infra_touched,
+ "Shared Sources Touched": s.shared_sources_touched,
+ "FE Modules Total": s.fe_modules_total,
+ "FE Modules Changed": s.fe_modules_changed,
+ "FE Modules Affected (Rules)": s.fe_modules_affected_rules,
+ "FE Modules Affected (Usage)": s.fe_modules_affected_usage,
+ "Unit Specs All": s.unit_specs_all,
+ "Unit Specs To Run (Rules)": s.unit_specs_to_run_rules,
+ "Unit Specs To Run (Usage)": s.unit_specs_to_run_usage,
+ "Loki Stories All": s.loki_stories_all,
+ "Loki Stories To Run (Rules)": s.loki_stories_to_run_rules,
+ "Loki Stories To Run (Usage)": s.loki_stories_to_run_usage,
+ "E2E Specs All": s.e2e_specs_all,
+ "E2E Specs To Run (Rules)": s.e2e_specs_to_run_rules,
+ "E2E Specs To Run (Usage)": s.e2e_specs_to_run_usage,
+ };
+
+ await uploadCsvToMb({
+ baseUrl: "https://stats.metabase.com",
+ tableId: TABLE_ID,
+ jsonData: [row],
+ mode: "append",
});
+ console.log("Data uploaded successfully");
+}
+
+// Best-effort: never let a telemetry failure turn the job red.
+main().catch((error) => {
+ console.error("Skipping stats upload after error:", error);
+ process.exit(0);
+});