Skip to content

Open-source status page, incident response and uptime monitoring

post: first 48h being public (#131)

Maximilian Kaske14fb70a1782c
diff --git a/apps/web/package.json b/apps/web/package.json
index 4fb5db44e..3f15eb7ca 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -57,6 +57,7 @@
"react-day-picker": "8.8.0",
"react-dom": "18.2.0",
"react-hook-form": "7.45.1",
+ "react-tweet": "^3.0.3",
"reading-time": "1.5.0",
"rehype-pretty-code": "0.10.0",
"resend": "0.15.3",
diff --git a/apps/web/public/assets/posts/blog-post-1.png b/apps/web/public/assets/posts/blog-post-1.png
deleted file mode 100644
index c4db5eec0..000000000
Binary files a/apps/web/public/assets/posts/blog-post-1.png and /dev/null differ
diff --git a/apps/web/public/assets/posts/the-first-48-hours/enough-gas-workspace-slug.png b/apps/web/public/assets/posts/the-first-48-hours/enough-gas-workspace-slug.png
new file mode 100644
index 000000000..191a5abf9
Binary files /dev/null and b/apps/web/public/assets/posts/the-first-48-hours/enough-gas-workspace-slug.png differ
diff --git a/apps/web/public/assets/posts/the-first-48-hours/github-trending.png b/apps/web/public/assets/posts/the-first-48-hours/github-trending.png
new file mode 100644
index 000000000..2f83fc0ce
Binary files /dev/null and b/apps/web/public/assets/posts/the-first-48-hours/github-trending.png differ
diff --git a/apps/web/public/assets/posts/the-first-48-hours/qstash-usage.png b/apps/web/public/assets/posts/the-first-48-hours/qstash-usage.png
new file mode 100644
index 000000000..500ad9b60
Binary files /dev/null and b/apps/web/public/assets/posts/the-first-48-hours/qstash-usage.png differ
diff --git a/apps/web/public/assets/posts/the-first-48-hours/vercel-usage.png b/apps/web/public/assets/posts/the-first-48-hours/vercel-usage.png
new file mode 100644
index 000000000..aad63772b
Binary files /dev/null and b/apps/web/public/assets/posts/the-first-48-hours/vercel-usage.png differ
diff --git a/apps/web/public/assets/posts/the-first-48-hours/wet-grandmother-workspace-slug.png b/apps/web/public/assets/posts/the-first-48-hours/wet-grandmother-workspace-slug.png
new file mode 100644
index 000000000..b1c96bfe4
Binary files /dev/null and b/apps/web/public/assets/posts/the-first-48-hours/wet-grandmother-workspace-slug.png differ
diff --git a/apps/web/src/app/blog/[slug]/page.tsx b/apps/web/src/app/blog/[slug]/page.tsx
index 361027a79..85b59b927 100644
--- a/apps/web/src/app/blog/[slug]/page.tsx
+++ b/apps/web/src/app/blog/[slug]/page.tsx
@@ -64,16 +64,22 @@ export default function PostPage({ params }: { params: { slug: string } }) {
return (
<>
<BackButton href="/blog" />
- <Shell>
+ <Shell className="sm:py-8 md:py-12">
<article className="grid gap-8">
- <div className="mx-auto grid max-w-prose gap-3">
- <h1 className="font-cal text-3xl">{post.title}</h1>
+ <div className="mx-auto grid w-full max-w-prose gap-3">
+ <h1 className="font-cal mb-5 text-3xl">{post.title}</h1>
<div className="border-border relative h-64 w-full overflow-hidden rounded-lg border">
- <Image
+ {/* <Image
src={post.image}
fill={true}
alt={post.title}
className="object-cover"
+ /> */}
+ {/* HOTFIX: plain `img` */}
+ <img
+ src={post.image}
+ alt={post.title}
+ className="h-full w-full object-cover"
/>
</div>
<p className="text-muted-foreground text-sm font-light">
diff --git a/apps/web/src/components/content/mdx-components.tsx b/apps/web/src/components/content/mdx-components.tsx
index 22f2bf874..16f487426 100644
--- a/apps/web/src/components/content/mdx-components.tsx
+++ b/apps/web/src/components/content/mdx-components.tsx
@@ -1,5 +1,7 @@
import * as React from "react";
import Link from "next/link";
+import type { TweetProps } from "react-tweet";
+import { Tweet } from "react-tweet";
export const components = {
a: ({
@@ -26,4 +28,11 @@ export const components = {
/>
);
},
+ Tweet: (props: TweetProps) => {
+ return (
+ <div data-theme="light" className="not-prose [&>div]:mx-auto">
+ <Tweet {...props} />
+ </div>
+ );
+ },
};
diff --git a/apps/web/src/components/content/mdx.tsx b/apps/web/src/components/content/mdx.tsx
index e9680f1e5..b681b79dc 100644
--- a/apps/web/src/components/content/mdx.tsx
+++ b/apps/web/src/components/content/mdx.tsx
@@ -11,7 +11,7 @@ export function Mdx({ code }: MdxProps) {
return (
// FIXME: weird behaviour when `prose-headings:font-cal` and on mouse movement font gets bigger
- <div className="prose prose-neutral dark:prose-invert prose-pre:border prose-pre:border-border prose-pre:rounded-lg">
+ <div className="prose prose-neutral dark:prose-invert prose-pre:border prose-pre:border-border prose-pre:rounded-lg prose-img:rounded-lg prose-img:border prose-img:border-border">
<MDXComponent components={{ ...components }} />
</div>
);
diff --git a/apps/web/src/content/posts/hello-world.mdx b/apps/web/src/content/posts/hello-world.mdx
deleted file mode 100644
index ec1ffb8fe..000000000
--- a/apps/web/src/content/posts/hello-world.mdx
+++ /dev/null
@@ -1,92 +0,0 @@
----
-title: Understanding Hoisting and Promises in JavaScript
-description:
- Understanding two important concepts in JavaScript - hoisting and promises.
-author:
- name: Nikhil Mohite
- url: https://nikhilmohite.com
-publishedAt: 2023-07-16
-image: /assets/posts/blog-post-1.png
----
-
-## Understanding Hoisting and Promises in JavaScript
-
-JavaScript is a versatile programming language that offers a wide range of
-features and functionalities. Two important concepts to grasp in JavaScript are
-hoisting and promises. Understanding these concepts can greatly enhance your
-ability to write efficient and reliable code. In this blog post, we will explore
-what hoisting and promises are, how they work, and provide some code examples to
-illustrate their usage.
-
-## Hoisting: Bringing Declarations to the Top
-
-Hoisting is a behavior in JavaScript where variable and function declarations
-are moved to the top of their containing scope during the compilation phase.
-This means that you can access and use variables and functions before they are
-actually declared in your code. However, it's important to note that only the
-declarations are hoisted, not the initializations.
-
-For example, consider the following code snippet:
-
-```js title="index.js" {} showLineNumbers
-console.log(name);
-```
-
-## Promises: Managing Asynchronous Operations
-
-Promises are a powerful concept introduced in ECMAScript 6 (ES6) that enable
-better management of asynchronous operations in JavaScript. They provide a
-cleaner and more structured approach to handling asynchronous tasks, such as
-network requests or file operations. A promise represents a value that may not
-be available yet, but will be resolved at some point in the future.
-
-Promises have three states: pending, fulfilled, or rejected. When a promise is
-pending, it means the asynchronous operation is still in progress. Once the
-operation is completed successfully, the promise is fulfilled and returns a
-value. On the other hand, if an error occurs during the operation, the promise
-is rejected, and an error object is returned.
-
-This code fetches data asynchronously using a promise in TypeScript, resolves
-the data after a delay, and handles the received data or any potential errors
-using then and catch blocks.
-
-```ts title="index.ts" {3, 16} showLineNumbers
-function getData(): Promise<string[]> {
- return new Promise<string[]>((resolve, reject) => {
- setTimeout(() => {
- const data: string[] = ["John", "Jane", "Bob"];
- if (data.length > 0) {
- resolve(data);
- } else {
- reject(new Error("No data available."));
- }
- }, 2000);
- });
-}
-
-getData()
- .then((data) => {
- console.log("Received data:", data);
- })
- .catch((error) => {
- console.error("Error:", error.message);
- });
-```
-
-## Advantages of Using Promises
-
-- Promises improve code readability and maintainability in JavaScript.
-- Promises simplify error handling in asynchronous code.
-- Promises help avoid <strong>"callback hell"</strong> by providing a more
- linear and sequential approach to asynchronous operations.
-
-## Conclusion
-
-In conclusion, hoisting and promises are important concepts in JavaScript that
-greatly contribute to writing efficient and reliable code. Hoisting allows
-variable and function declarations to be moved to the top of their scope,
-enabling early access to these entities. Promises, on the other hand, provide a
-structured and readable approach to managing asynchronous operations, enhancing
-code maintainability and error handling.
-
-[test link](https://openstatus.dev)
diff --git a/apps/web/src/content/posts/the-first-48-hours.mdx b/apps/web/src/content/posts/the-first-48-hours.mdx
new file mode 100644
index 000000000..d845d0f84
--- /dev/null
+++ b/apps/web/src/content/posts/the-first-48-hours.mdx
@@ -0,0 +1,109 @@
+---
+title: 48 hours of public OpenStatus
+description: The numbers, limits we faced and consequences we have taken.
+author:
+ name: Maximilian Kaske
+ url: https://twitter.com/mxkaske
+publishedAt: 2023-08-02
+image: /assets/posts/first-48h/github-trending.png
+---
+
+## 48 hours of Rollercoaster 🎒
+
+The past two days following the launch have been incredibly hectic. The level of
+interest we received exceeded our expectations. We were even trending on GitHub.
+The number of users and created monitors far surpassed what we anticipated. We
+sincerely appreciate your support, as it further motivates us. The raw
+statistics are astonishing:
+
+- Monitors: 315
+- Users: 418
+- Status pages: 174
+
+With a large user base comes great responsibility. Thanks to the high volume of
+users, we discovered several flaws in our system. We promptly addressed the
+issues and took action.
+
+## Limits we encountered 😱
+
+We conducted a DDOS attack on Vercel's firewalls from Upstash servers because we
+where pining every single vercel region for every monitor (18 \* 315 = 5.670)
+more or less at once. Before considering the removal of
+[QStash](http://upstash.com/qstash?ref=openstatus), we implemented a hotfix to
+introduce random delays of 0 to 180 seconds between each ping. This prevented us
+from overwhelming Vercel with simultaneous requests, significantly reducing the
+number of retries on QStash. As a result of these measures, we now have a
+randomized check interval of 0 to 90 seconds.
+
+For the time being, we will keep it to ensure that every check is successfully
+processed.
+
+![Qstash Usage](/assets/posts/first-48h/qstash-usage.png)
+
+Moreover, our actions caused the `/monitor/[id]/data` data-table to break. We
+were storing the metadata `res.text()` in Tinybird and retrieving it with the
+query `SELECT * from monitors WHERE ....` This means that if the text is in HTML
+format, we store the HTML content of the page, which is acceptable. However,
+this led to quickly exceeding the 100MB limit for the result length when
+accessing monitor data. Now, we are only querying the necessary data from
+tinybird: `SELECT latency, timestamp, url, ... from monitors WHERE ...`. Despite
+this, we continue to ingest the data and will only request it in the future when
+you click on the "View metadata" action.
+
+And of course no need to mention all the little hotfixed we had to make here and
+there.
+
+### What would it cost us? πŸ’Έ
+
+Let's break the current numbers a bit more down if we would have kept pinging
+all the regions:
+
+- Number of checks every 10min: 315 \* 18 regions = 5.670
+- Number of checks every 1h: 5.670 \* 6 = 34.020
+- Number of checks every day: 34.020 \* 24 = 816.480
+
+![Vercel Usage](/assets/posts/first-48h/vercel-usage.png)
+
+> We have resolved an issue with regex invocation in the edge middleware.
+> Previously, it was being called whenever an API endpoint was called.
+
+Under Vercel's Pro plan, we have 1 million edge function executions included,
+and for each additional 1 million, there is a
+$2 charge. Because the numbers are constant, we would have payed
+1$/100k Qstash
+messages which would lead us to a approx. **10$/day** for the current free
+tiers.
+
+### Actions taken πŸ‘·
+
+But due to the high cost of our current infrastructure, **we have downgraded all
+monitors to a single region** (`/regions/auto`, a which will take a random
+vercel region) in order to stay within a reasonable budget, by dividing it
+by 18. You can edit the region for each monitor. We apologize for any
+inconvenience caused and will update you on the new conditions once everything
+is normalized.
+
+A big appreciation to [@chronark\_](https://twitter.com/chronark_) who joined us
+on a call to discuss potential solutions and for sharing his knowledge about his
+own service, [planetfall.io](https://planetfall.io).
+
+Also, Guilherme had a very good comment:
+
+<Tweet id="1686482013685940224" />
+
+As a result of this incident, we will be reevaluating the prices and plans
+displayed on our homepage.
+
+Join [Discord](https://openstatus.dev/discord) if you want to learn more or just
+have a chat!
+
+Again, thank you for all your support and your understanding. πŸ™
+
+---
+
+P.S. We are generating your workspace-slug by merging two random names. Create
+an account on [openstatus](https://openstatus.dev) and share yours on
+[Discord](https://openstatus.dev/discord) - it's fun to watch! 🍿
+
+![wet-grandmother slug](/assets/posts/first-48h/wet-grandmother-workspace-slug.png)
+![enough-gas slug](/assets/posts/first-48h/enough-gas-workspace-slug.png)
diff --git a/apps/web/src/middleware.ts b/apps/web/src/middleware.ts
index 9d63b8738..cd4274890 100644
--- a/apps/web/src/middleware.ts
+++ b/apps/web/src/middleware.ts
@@ -104,7 +104,7 @@ export default authMiddleware({
export const config = {
matcher: [
- "/((?!api|_next/static|_next/image|favicon.ico).*)",
+ "/((?!api|assets|_next/static|_next/image|favicon.ico|sitemap.xml|robots.txt).*)",
"/",
"/(api/webhook|api/trpc)(.*)",
"/(!api/checker/:path*|!api/og|!api/ping)",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 5eb9a0731..32b667efd 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -185,6 +185,9 @@ importers:
react-hook-form:
specifier: 7.45.1
version: 7.45.1([email protected])
+ react-tweet:
+ specifier: ^3.0.3
reading-time:
specifier: 1.5.0
version: 1.5.0
@@ -2728,7 +2731,7 @@ packages:
'@peculiar/asn1-schema': 2.3.6
'@peculiar/json-schema': 1.1.12
pvtsutils: 1.3.2
- tslib: 2.4.1
+ tslib: 2.6.1
webcrypto-core: 1.7.7
dev: false
@@ -10384,6 +10387,20 @@ packages:
tslib: 2.6.1
dev: false
+ resolution: {integrity: sha512-dKPpCCGGo07hV8QtOthAwVbG/VfDA22kg5ny53UHnYYMR18gv7hDYiT9ZjMS4hWLzEObz++e616Sf4qLo9hhag==}
+ peerDependencies:
+ react: '>= 18.0.0'
+ react-dom: '>= 18.0.0'
+ dependencies:
+ '@swc/helpers': 0.5.1
+ clsx: 1.2.1
+ date-fns: 2.30.0
+ react: 18.2.0
+ react-dom: 18.2.0([email protected])
+ swr: 2.2.0([email protected])
+ dev: false
+
resolution: {integrity: sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==}
engines: {node: '>=0.10.0'}
@@ -11133,6 +11150,15 @@ packages:
react: 18.2.0
dev: false
+ resolution: {integrity: sha512-AjqHOv2lAhkuUdIiBu9xbuettzAzWXmCEcLONNKJRba87WAefz8Ca9d6ds/SzrPc235n1IxWYdhJ2zF3MNUaoQ==}
+ peerDependencies:
+ react: ^16.11.0 || ^17.0.0 || ^18.0.0
+ dependencies:
+ react: 18.2.0
+ use-sync-external-store: 1.2.0([email protected])
+ dev: false
+
resolution: {integrity: sha512-L1dapNV6vu2s/4Sputv8xGsCdAVlb5nRDMFU/E27D44l5U6cw1g0dGd45uLc+OXjNMmF4ntiMdCimzcjFKQI8Q==}
engines: {node: ^14.18.0 || >=16.0.0}