Skip to content

Open-source status page, incident response and uptime monitoring

πŸ” add retry for screenshot service (#849)

Thibault Le Ouay5a8c106849a7
diff --git a/apps/screenshot-service/fly.toml b/apps/screenshot-service/fly.toml
index 1dd4d0f3d..08d585f9d 100644
--- a/apps/screenshot-service/fly.toml
+++ b/apps/screenshot-service/fly.toml
@@ -22,8 +22,8 @@ primary_region = 'ams'
[http_service.concurrency]
type = 'requests'
- hard_limit = 1000
- soft_limit = 500
+ hard_limit = 4
+ soft_limit = 2
[[http_service.checks]]
interval = '15s'
diff --git a/apps/server/.env.example b/apps/server/.env.example
index 7161b06ae..00d498551 100644
--- a/apps/server/.env.example
+++ b/apps/server/.env.example
@@ -10,3 +10,4 @@ TWILLIO_AUTH_TOKEN=your_auth_token
TWILLIO_ACCOUNT_ID=your_account_id
SCREENSHOT_SERVICE_URL=http://your.endpoint
+QSTASH_TOKEN=your_token
diff --git a/apps/server/package.json b/apps/server/package.json
index 000f51a86..fb3f094e6 100644
--- a/apps/server/package.json
+++ b/apps/server/package.json
@@ -27,6 +27,7 @@
"@openstatus/utils": "workspace:*",
"@t3-oss/env-core": "0.7.1",
"@unkey/api": "0.16.0",
+ "@upstash/qstash": "2.1.8",
"hono": "4.0.0",
"nanoid": "5.0.2",
"validator": "13.11.0",
diff --git a/apps/server/src/checker/index.ts b/apps/server/src/checker/index.ts
index a4cd177ce..858b767b5 100644
--- a/apps/server/src/checker/index.ts
+++ b/apps/server/src/checker/index.ts
@@ -1,5 +1,6 @@
import { Hono } from "hono";
import { z } from "zod";
+import { Client } from "@upstash/qstash";
import { and, db, eq, isNull, schema } from "@openstatus/db";
import { incidentTable } from "@openstatus/db/src/schema";
@@ -58,8 +59,8 @@ checkerRoute.post("/updateStatus", async (c) => {
and(
eq(incidentTable.monitorId, Number(monitorId)),
isNull(incidentTable.resolvedAt),
- isNull(incidentTable.acknowledgedAt),
- ),
+ isNull(incidentTable.acknowledgedAt)
+ )
)
.get();
@@ -99,7 +100,7 @@ checkerRoute.post("/updateStatus", async (c) => {
const numberOfRegions = monitor.regions.length;
console.log(
- `πŸ€“ MonitorID ${monitorId} incident current affected ${nbAffectedRegion} total region ${numberOfRegions}`,
+ `πŸ€“ MonitorID ${monitorId} incident current affected ${nbAffectedRegion} total region ${numberOfRegions}`
);
// If the number of affected regions is greater than half of the total region, we trigger the alerting
// 4 of 6 monitor need to fail to trigger an alerting
@@ -113,8 +114,8 @@ checkerRoute.post("/updateStatus", async (c) => {
eq(incidentTable.monitorId, Number(monitorId)),
isNull(incidentTable.resolvedAt),
isNull(incidentTable.acknowledgedAt),
- eq(incidentTable.startedAt, new Date(cronTimestamp)),
- ),
+ eq(incidentTable.startedAt, new Date(cronTimestamp))
+ )
)
.get();
@@ -191,7 +192,7 @@ checkerRoute.post("/updateStatus", async (c) => {
const numberOfRegions = monitor.regions.length;
console.log(
- `πŸ€“ MonitorId ${monitorId} recovering incident current ${nbAffectedRegion} total region ${numberOfRegions}`,
+ `πŸ€“ MonitorId ${monitorId} recovering incident current ${nbAffectedRegion} total region ${numberOfRegions}`
);
// // If the number of affected regions is greater than half of the total region, we trigger the alerting
// // 4 of 6 monitor need to fail to trigger an alerting
@@ -203,8 +204,8 @@ checkerRoute.post("/updateStatus", async (c) => {
and(
eq(incidentTable.monitorId, Number(monitorId)),
isNull(incidentTable.resolvedAt),
- isNull(incidentTable.acknowledgedAt),
- ),
+ isNull(incidentTable.acknowledgedAt)
+ )
)
.get();
if (incident) {
@@ -258,16 +259,20 @@ const triggerScreenshot = async ({
data: z.infer<typeof payload>;
}) => {
console.log(` πŸ“Έ taking screenshot for incident ${data.incidentId}`);
- await fetch(env.SCREENSHOT_SERVICE_URL, {
+
+ const client = new Client({ token: env.QSTASH_TOKEN });
+
+ await client.publishJSON({
+ url: env.SCREENSHOT_SERVICE_URL,
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Basic ${env.CRON_SECRET}`,
},
- body: JSON.stringify({
+ body: {
url: data.url,
incidentId: data.incidentId,
kind: data.kind,
- }),
+ },
});
};
diff --git a/apps/server/src/env.ts b/apps/server/src/env.ts
index 7fa3ca288..255e074c1 100644
--- a/apps/server/src/env.ts
+++ b/apps/server/src/env.ts
@@ -15,6 +15,7 @@ export const env = createEnv({
JITSU_WRITE_KEY: z.string().optional(),
JITSU_HOST: z.string().optional(),
SCREENSHOT_SERVICE_URL: z.string(),
+ QSTASH_TOKEN: z.string(),
},
/**
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 9a873238e..7be953075 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -190,6 +190,9 @@ importers:
'@unkey/api':
specifier: 0.16.0
version: 0.16.0
+ '@upstash/qstash':
+ specifier: 2.1.8
+ version: 2.1.8
hono:
specifier: 4.0.0
version: 4.0.0