Skip to content

Open-source status page, incident response and uptime monitoring

Improve retry (#390)

Thibault Le Ouay6728033537b7
diff --git a/apps/server/log/fly.toml b/apps/server/log/fly.toml
new file mode 100644
index 000000000..376c66140
--- /dev/null
+++ b/apps/server/log/fly.toml
@@ -0,0 +1,18 @@
+# fly.toml app configuration file generated for openstatus-log on 2023-10-19T00:44:54+02:00
+#
+# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
+#
+
+app = "openstatus-log"
+primary_region = "ams"
+
+[build]
+ image = "ghcr.io/superfly/fly-log-shipper:latest"
+
+[http_service]
+ internal_port = 8686
+ force_https = true
+ auto_stop_machines = true
+ auto_start_machines = true
+ min_machines_running = 1
+ processes = ["app"]
diff --git a/apps/server/src/checker/checker.ts b/apps/server/src/checker/checker.ts
index c3a93864a..7ff81f5eb 100644
--- a/apps/server/src/checker/checker.ts
+++ b/apps/server/src/checker/checker.ts
@@ -49,12 +49,27 @@ export const checker = async (data: Payload) => {
});
}
} else {
- console.log(`error for ${data.url} with status ${res.status}`);
- if (data?.status === "active") {
- await updateMonitorStatus({
- monitorId: data.monitorId,
- status: "error",
- });
+ console.log(`first retry for ${data.url} with status ${res.status}`);
+ const startTime = Date.now();
+ const retry = await ping(data);
+ const endTime = Date.now();
+ const latency = endTime - startTime;
+ if (retry.ok) {
+ await monitor({ monitorInfo: data, latency, statusCode: res.status });
+ if (data?.status === "error") {
+ await updateMonitorStatus({
+ monitorId: data.monitorId,
+ status: "active",
+ });
+ }
+ } else {
+ console.log(`error for ${data.url} with status ${res.status}`);
+ if (data?.status === "active") {
+ await updateMonitorStatus({
+ monitorId: data.monitorId,
+ status: "error",
+ });
+ }
}
}
};
diff --git a/apps/web/src/app/api/checker/cron/_cron.ts b/apps/web/src/app/api/checker/cron/_cron.ts
index fb31c4fd3..fc7ea284d 100644
--- a/apps/web/src/app/api/checker/cron/_cron.ts
+++ b/apps/web/src/app/api/checker/cron/_cron.ts
@@ -48,54 +48,28 @@ export const cron = async ({
monitorId: row.id,
});
- if (row.regions.length === 0 || row.regions.includes("auto")) {
+ for (const region of flyRegions) {
const payload: z.infer<typeof payloadSchema> = {
workspaceId: String(row.workspaceId),
- method: row.method || "GET",
monitorId: String(row.id),
url: row.url,
- headers: row.headers,
- body: row.body,
+ method: row.method || "GET",
cronTimestamp: timestamp,
- status: row.status,
+ body: row.body,
+ headers: row.headers,
pageIds: allPages.map((p) => String(p.pageId)),
+ status: row.status,
};
- // TODO: fetch + try - catch + retry once
const result = c.publishJSON({
url: `https://api.openstatus.dev/checker`,
body: payload,
delay: Math.random() * 90,
headers: {
- "Upstash-Forward-fly-prefer-region": "ams",
+ "Upstash-Forward-fly-prefer-region": region,
},
});
allResult.push(result);
- } else {
- const regions = row.regions;
- for (const region of regions) {
- const payload: z.infer<typeof payloadSchema> = {
- workspaceId: String(row.workspaceId),
- monitorId: String(row.id),
- url: row.url,
- method: row.method || "GET",
- cronTimestamp: timestamp,
- body: row.body,
- headers: row.headers,
- pageIds: allPages.map((p) => String(p.pageId)),
- status: row.status,
- };
-
- const result = c.publishJSON({
- url: `https://api.openstatus.dev/checker`,
- body: payload,
- delay: Math.random() * 90,
- headers: {
- "Upstash-Forward-fly-prefer-region": region,
- },
- });
- allResult.push(result);
- }
}
}
// our first legacy monitor