The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Make test stats upload more robust (#77258)
Fraser384f6fb9b76d
diff --git a/.github/scripts/csv-to-mb.js b/.github/scripts/csv-to-mb.jsindex 5f5de281371..a9f024db823 100644--- a/.github/scripts/csv-to-mb.js+++ b/.github/scripts/csv-to-mb.js@@ -30,7 +30,7 @@ class UploadError extends Error {}}-async function attemptUpload(url, jsonData) {+async function attemptUpload(url, jsonData, timeoutMs) {let response;try {response = await fetch(url, {@@ -39,7 +39,11 @@ async function attemptUpload(url, jsonData) {"x-api-key": process.env.API_KEY},// Rebuilt per attempt: a consumed request body can't be reused.- body: jsonToCsvFormData(jsonData)+ body: jsonToCsvFormData(jsonData),+ // Without this the request can hang indefinitely on a stalled server,+ // eating the whole job timeout. An abort surfaces below as a retryable+ // network error, so it retries with backoff instead.+ signal: AbortSignal.timeout(timeoutMs)});} catch (networkError) {throw new UploadError(networkError.message, { retryable: true });@@ -61,16 +65,17 @@ const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));/*** Uploads jsonData as CSV, retrying transient API errors with exponential* backoff. `retries` is the number of extra attempts after the first (so the- * default 2 means up to 3 tries). Throws the last error once attempts are- * exhausted or the error is non-retryable.+ * default 2 means up to 3 tries). `timeoutMs` is a per-attempt total deadline+ * (default 60s) that aborts a stalled request. Throws the last error once+ * attempts are exhausted or the error is non-retryable.*/-async function uploadCsvToMb({ baseUrl, tableId, jsonData, mode = 'append', retries = 2, retryDelayMs = 1000 }) {+async function uploadCsvToMb({ baseUrl, tableId, jsonData, mode = 'append', retries = 2, retryDelayMs = 1000, timeoutMs = 60_000 }) {const operation = mode === 'replace' ? 'replace-csv' : 'append-csv';const url = `${baseUrl}/api/table/${tableId}/${operation}`;for (let attempt = 0; ; attempt++) {try {- await attemptUpload(url, jsonData);+ await attemptUpload(url, jsonData, timeoutMs);return { success: true };} catch (error) {if (!error.retryable || attempt >= retries) {diff --git a/.github/scripts/upload-affected-tests-stats.js b/.github/scripts/upload-affected-tests-stats.jsindex 4c9731cec31..7f4f98ced4a 100644--- a/.github/scripts/upload-affected-tests-stats.js+++ b/.github/scripts/upload-affected-tests-stats.js@@ -7,48 +7,58 @@ const { uploadCsvToMb } = require("./csv-to-mb.js");// "FE Affected Tests" uploaded table on stats.metabase.com.const TABLE_ID = 81818;-const s = JSON.parse(process.env.STATS_JSON);-console.log("stats", s);+async function main() {+ // create-test-plan doesn't always produce stats (e.g. on release branches+ // running an older planner), leaving STATS_JSON empty. Nothing to upload.+ const raw = (process.env.STATS_JSON || "").trim();+ if (!raw || raw === "null") {+ console.log("No STATS_JSON to upload; skipping.");+ return;+ }-const row = {- Date: new Date().toISOString(),- // "pr_update" or "merge_to_master". ("Trigger" is a reserved SQL word.)- "Triggered By": process.env.TRIGGERED_BY,- PR: Number(process.env.PR_NUMBER),- "Head SHA": process.env.HEAD_SHA,- "Base SHA": process.env.BASE_SHA,- "FE Files Changed": s.fe_files_changed,- "FE Files Total": s.fe_files_total,- "BE Files Changed": s.be_files_changed,- "BE Files Total": s.be_files_total,- "Unit Infra Touched": s.unit_infra_touched,- "Loki Infra Touched": s.loki_infra_touched,- "Shared Sources Touched": s.shared_sources_touched,- "FE Modules Total": s.fe_modules_total,- "FE Modules Changed": s.fe_modules_changed,- "FE Modules Affected (Rules)": s.fe_modules_affected_rules,- "FE Modules Affected (Usage)": s.fe_modules_affected_usage,- "Unit Specs All": s.unit_specs_all,- "Unit Specs To Run (Rules)": s.unit_specs_to_run_rules,- "Unit Specs To Run (Usage)": s.unit_specs_to_run_usage,- "Loki Stories All": s.loki_stories_all,- "Loki Stories To Run (Rules)": s.loki_stories_to_run_rules,- "Loki Stories To Run (Usage)": s.loki_stories_to_run_usage,- "E2E Specs All": s.e2e_specs_all,- "E2E Specs To Run (Rules)": s.e2e_specs_to_run_rules,- "E2E Specs To Run (Usage)": s.e2e_specs_to_run_usage,-};+ const s = JSON.parse(raw);+ console.log("stats", s);-uploadCsvToMb({- baseUrl: "https://stats.metabase.com",- tableId: TABLE_ID,- jsonData: [row],- mode: "append",-})- .then(() => {- console.log("Data uploaded successfully");- })- .catch((error) => {- console.error("Error uploading data:", error);- process.exit(1);+ const row = {+ Date: new Date().toISOString(),+ // "pr_update" or "merge_to_master". ("Trigger" is a reserved SQL word.)+ "Triggered By": process.env.TRIGGERED_BY,+ PR: Number(process.env.PR_NUMBER),+ "Head SHA": process.env.HEAD_SHA,+ "Base SHA": process.env.BASE_SHA,+ "FE Files Changed": s.fe_files_changed,+ "FE Files Total": s.fe_files_total,+ "BE Files Changed": s.be_files_changed,+ "BE Files Total": s.be_files_total,+ "Unit Infra Touched": s.unit_infra_touched,+ "Loki Infra Touched": s.loki_infra_touched,+ "Shared Sources Touched": s.shared_sources_touched,+ "FE Modules Total": s.fe_modules_total,+ "FE Modules Changed": s.fe_modules_changed,+ "FE Modules Affected (Rules)": s.fe_modules_affected_rules,+ "FE Modules Affected (Usage)": s.fe_modules_affected_usage,+ "Unit Specs All": s.unit_specs_all,+ "Unit Specs To Run (Rules)": s.unit_specs_to_run_rules,+ "Unit Specs To Run (Usage)": s.unit_specs_to_run_usage,+ "Loki Stories All": s.loki_stories_all,+ "Loki Stories To Run (Rules)": s.loki_stories_to_run_rules,+ "Loki Stories To Run (Usage)": s.loki_stories_to_run_usage,+ "E2E Specs All": s.e2e_specs_all,+ "E2E Specs To Run (Rules)": s.e2e_specs_to_run_rules,+ "E2E Specs To Run (Usage)": s.e2e_specs_to_run_usage,+ };++ await uploadCsvToMb({+ baseUrl: "https://stats.metabase.com",+ tableId: TABLE_ID,+ jsonData: [row],+ mode: "append",});+ console.log("Data uploaded successfully");+}++// Best-effort: never let a telemetry failure turn the job red.+main().catch((error) => {+ console.error("Skipping stats upload after error:", error);+ process.exit(0);+});