The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
add: blog post about symbolication (#222)
narrow-adventure31d00a0bed4c
diff --git a/diagrams/blog/system-design-of-a-symbolicator/cache-in-memory.dot b/diagrams/blog/system-design-of-a-symbolicator/cache-in-memory.dotnew file mode 100644index 00000000..e768b002--- /dev/null+++ b/diagrams/blog/system-design-of-a-symbolicator/cache-in-memory.dot@@ -0,0 +1,37 @@+// Stage 2 "build once, cache in memory" flow. Render:+// dot -Tsvg cache-in-memory.dot -o cache-in-memory.svg+// Palette matches the Traceway dark theme (app/globals.css).+digraph cache_in_memory {+ bgcolor="transparent";+ rankdir=TB;+ nodesep=0.34;+ ranksep=0.44;+ pad=0.2;+ fontname="Helvetica";++ node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=14,+ penwidth=1.3, margin="0.30,0.18",+ color="#283041", fillcolor="#151a24", fontcolor="#f4f6fb"];+ edge [color="#5a6374", penwidth=1.4, arrowsize=0.85,+ fontname="Helvetica", fontsize=11, fontcolor="#8a93a6"];++ req [label=<<b>incoming stack frame</b><br/><font face="Menlo" color="#8a93a6" point-size="12">app.min.js:1:5114</font>>,+ fillcolor="#0f131c"];+ cache [label=<<b>look up the file in the cache</b><br/><font color="#8fe6ff" point-size="12">in-memory, LRU, keyed by file</font>>,+ color="#00d4ff", fillcolor="#08161c"];+ build [label=<<b>parse → build the .tw index</b><br/><font color="#ff9a9d" point-size="12">slow, but only once per file</font>>,+ color="#ff5a5f", fillcolor="#1b1115"];+ store [label=<<b>keep the .tw in the cache</b><br/><font color="#8fe6ff" point-size="12">reused by every later request</font>>,+ color="#00d4ff", fillcolor="#08161c"];+ look [label=<<b>search the cached index</b><br/><font color="#7ff0c8" point-size="12">fast: 44 ns, no parsing</font>>,+ color="#22e0a8", fillcolor="#0c1714"];+ out [label=<<b>resolved frame</b><br/><font face="Menlo" color="#b9a7ff" point-size="12">checkout.js:48 · validateCheckout</font>>,+ color="#7c5cff", fillcolor="#15121f"];++ req -> cache;+ cache -> build [label=" miss (first time for this file)", color="#5a6374"];+ cache -> look [constraint=false, tailport=e, headport=e,+ label="hit\l(almost every\lrequest)\l", color="#22e0a8", fontcolor="#7ff0c8"];+ build -> store -> look;+ look -> out;+}diff --git a/diagrams/blog/system-design-of-a-symbolicator/parse-on-every-request.dot b/diagrams/blog/system-design-of-a-symbolicator/parse-on-every-request.dotnew file mode 100644index 00000000..a20a7059--- /dev/null+++ b/diagrams/blog/system-design-of-a-symbolicator/parse-on-every-request.dot@@ -0,0 +1,34 @@+// Stage 1 "dumb version" flow. Render:+// dot -Tsvg parse-on-every-request.dot -o parse-on-every-request.svg+// Palette matches the Traceway dark theme (app/globals.css).+digraph parse_every_request {+ bgcolor="transparent";+ rankdir=TB;+ nodesep=0.32;+ ranksep=0.42;+ pad=0.2;+ fontname="Helvetica";++ node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=14,+ penwidth=1.3, margin="0.30,0.18",+ color="#283041", fillcolor="#151a24", fontcolor="#f4f6fb"];+ edge [color="#5a6374", penwidth=1.4, arrowsize=0.85,+ fontname="Helvetica", fontsize=10, fontcolor="#8a93a6"];++ req [label=<<b>incoming stack frame</b><br/><font face="Menlo" color="#8a93a6" point-size="12">app.min.js:1:5114</font>>,+ fillcolor="#0f131c"];+ load [label=<<b>1. load the debug artifact</b><br/><font color="#8a93a6" point-size="12">source map + bundle · .dSYM · .symbols</font>>];+ parse [label=<<b>2. parse → build a lookup index</b><br/><font color="#ff9a9d" point-size="12">slow: reads the whole file, thousands of allocations</font>>,+ color="#ff5a5f", fillcolor="#1b1115"];+ search [label=<<b>3. search the index for the frame</b><br/><font color="#7ff0c8" point-size="12">fast: a single lookup</font>>,+ color="#22e0a8", fillcolor="#0c1714"];+ ret [label=<<b>4. return the result, then drop the index</b>>];+ out [label=<<b>resolved frame</b><br/><font face="Menlo" color="#b9a7ff" point-size="12">checkout.js:48 · validateCheckout</font>>,+ color="#7c5cff", fillcolor="#15121f"];++ req -> load -> parse -> search -> ret -> out;++ ret -> load [constraint=false, tailport=e, headport=e,+ style=dashed, color="#ff5a5f", fontcolor="#ff8a8d",+ label=" next request:\l nothing kept,\l reload + reparse\l"];+}diff --git a/diagrams/blog/system-design-of-a-symbolicator/store-and-mmap.dot b/diagrams/blog/system-design-of-a-symbolicator/store-and-mmap.dotnew file mode 100644index 00000000..893b580a--- /dev/null+++ b/diagrams/blog/system-design-of-a-symbolicator/store-and-mmap.dot@@ -0,0 +1,41 @@+// Stage 3 "store it and mmap it" cascade. Render:+// dot -Tsvg store-and-mmap.dot -o store-and-mmap.svg+// Palette matches the Traceway dark theme (app/globals.css).+digraph store_and_mmap {+ bgcolor="transparent";+ rankdir=TB;+ nodesep=0.34;+ ranksep=0.44;+ pad=0.2;+ fontname="Helvetica";++ node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=14,+ penwidth=1.3, margin="0.30,0.18",+ color="#283041", fillcolor="#151a24", fontcolor="#f4f6fb"];+ edge [color="#5a6374", penwidth=1.4, arrowsize=0.85,+ fontname="Helvetica", fontsize=11, fontcolor="#8a93a6"];++ req [label=<<b>incoming stack frame</b><br/><font face="Menlo" color="#8a93a6" point-size="12">app.min.js:1:5114</font>>,+ fillcolor="#0f131c"];+ local [label=<<b>tier 1 · local disk</b><br/><font color="#8a93a6" point-size="12">is the .tw file on this box?</font>>];+ obj [label=<<b>tier 2 · object storage</b><br/><font color="#8a93a6" point-size="12">is a prebuilt .tw stored?</font>>];+ rebuild [label=<<b>tier 3 · build it once</b><br/><font color="#ff9a9d" point-size="12">first crash of a brand-new release</font>>,+ color="#ff5a5f", fillcolor="#1b1115"];+ fetch [label=<<b>download the .tw, write to disk</b><br/><font color="#8fe6ff" point-size="12">still no parsing</font>>,+ color="#00d4ff", fillcolor="#08161c"];+ back [label=<<b>write the .tw to object storage</b><br/><font color="#8fe6ff" point-size="12">every replica is warm after this</font>>,+ color="#00d4ff", fillcolor="#08161c"];+ mmap [label=<<b>mmap the .tw, search in place</b><br/><font color="#7ff0c8" point-size="12">fast: served from the OS page cache</font>>,+ color="#22e0a8", fillcolor="#0c1714"];+ out [label=<<b>resolved frame</b><br/><font face="Menlo" color="#b9a7ff" point-size="12">checkout.js:48 · validateCheckout</font>>,+ color="#7c5cff", fillcolor="#15121f"];++ req -> local;+ local -> mmap [label=" hit (hot path)", color="#22e0a8", fontcolor="#7ff0c8"];+ local -> obj [label=" miss", color="#5a6374"];+ obj -> fetch [label=" hit", color="#5a6374"];+ fetch -> mmap;+ obj -> rebuild [label=" miss (cold-cold)", color="#5a6374"];+ rebuild -> back -> mmap;+ mmap -> out;+}diff --git a/diagrams/blog/system-design-of-a-symbolicator/tw-format.dot b/diagrams/blog/system-design-of-a-symbolicator/tw-format.dotnew file mode 100644index 00000000..e41d77b2--- /dev/null+++ b/diagrams/blog/system-design-of-a-symbolicator/tw-format.dot@@ -0,0 +1,38 @@+// The .tw resolver format (JS encoding). Render:+// dot -Tsvg tw-format.dot -o tw-format.svg+// Palette matches the Traceway dark theme (app/globals.css).+digraph tw_format {+ bgcolor="transparent";+ pad=0.15;+ node [shape=plaintext, fontname="Helvetica"];++ tw [label=<+ <table border="1" color="#283041" cellborder="0" cellspacing="0" cellpadding="9" bgcolor="#0d111a">+ <tr><td colspan="3" align="left"><font color="#f4f6fb" point-size="15"><b>the .tw resolver format</b></font> <font color="#5a6374" point-size="11">flat · little-endian · memory-mapped</font></td></tr>+ <hr/>+ <tr>+ <td align="left" bgcolor="#11161f"><font color="#c9d0dd" point-size="14"><b>header</b></font></td>+ <td align="left" bgcolor="#11161f"><font face="Menlo" color="#8a93a6" point-size="13">magic "TWSM" · version · table counts</font></td>+ <td align="right" bgcolor="#11161f"><font color="#5a6374" point-size="12">24 B</font></td>+ </tr>+ <hr/>+ <tr>+ <td align="left" bgcolor="#0e1322"><font color="#b9a7ff" point-size="14"><b>token[ ]</b></font></td>+ <td align="left" bgcolor="#0e1322"><font face="Menlo" color="#c9d0dd" point-size="13">genLine genCol srcLine srcCol fileIdx fnIdx</font><br align="left"/><font color="#5a6374" point-size="11">sorted by (genLine, genCol) for binary search</font></td>+ <td align="right" bgcolor="#0e1322"><font color="#8a93a6" point-size="12">24 B<br/>each</font></td>+ </tr>+ <hr/>+ <tr>+ <td align="left" bgcolor="#0a1620"><font color="#8fe6ff" point-size="14"><b>file strings</b></font></td>+ <td align="left" bgcolor="#0a1620"><font face="Menlo" color="#c9d0dd" point-size="13">offsets[ ] + blob</font></td>+ <td align="right" bgcolor="#0a1620"><font color="#5a6374" point-size="12">interned</font></td>+ </tr>+ <hr/>+ <tr>+ <td align="left" bgcolor="#0a1620"><font color="#8fe6ff" point-size="14"><b>fn strings</b></font></td>+ <td align="left" bgcolor="#0a1620"><font face="Menlo" color="#c9d0dd" point-size="13">offsets[ ] + blob</font></td>+ <td align="right" bgcolor="#0a1620"><font color="#5a6374" point-size="12">interned</font></td>+ </tr>+ </table>+ >];+}diff --git a/website/content/blog/system-design-of-a-symbolicator.mdx b/website/content/blog/system-design-of-a-symbolicator.mdxnew file mode 100644index 00000000..83dc4852--- /dev/null+++ b/website/content/blog/system-design-of-a-symbolicator.mdx@@ -0,0 +1,340 @@+---+title: "System Design: Building a world-class symbolicator"+date: "2026-06-20"+category: "engineering"+description: "The shape of your data can make a service fast or slow. A walk from the slow version to the fast one, built on a real example: turning unreadable production crashes back into readable code."+---++> Premature optimization is the root of all evil.++Knuth wrote that in 1974, and it's still true today. Quality software is built by understanding the problem first, then improving the solution pragmatically, one measured step at a time. That's the structure of this post.++When you ship JavaScript to production, a build step shrinks it into one file, the **bundle**. A function named `validateCheckout` becomes `n`, whitespace vanishes, and the whole program collapses onto a single line. This is **minification**. Great for download size, miserable the moment something breaks.++Because when that code crashes, the report you get is a **stack trace**: the chain of function calls that led to the crash. Each call is a **frame**, printed as a line like `at n (app.min.js:1:5114)`. A spot in a file nobody can read, under a name that used to mean something. Stack a dozen of those and you can't act on it.++Turning that gibberish back into `validateCheckout (checkout.js:48:10)` has a name: **symbolication**. The service that does it is a **symbolicator**. You feed it the minified stack trace plus a build artifact or two, chiefly the **source map**: the companion file your build tool emits that records where each scrap of minified code came from. It hands you back real files, real line numbers, real function names. Every serious error tracker has one.++I've taken apart the *algorithm* inside a symbolicator before, by hand, twice: once for [JavaScript source maps](/blog/deep-dive-into-the-sourcemaps) and once for an [obfuscated Flutter crash](/blog/flutter-symbolication). You don't need either to follow this. This post is about the part both skipped. It's the part that decides whether the thing works in production: the **system design**.++The job sounds simple. Run that algorithm for every crash, from every release, of every customer. Do it fast enough that symbolication never makes an event show up late. The simple version is painfully slow. The road from slow to fast runs through caching, binary file layout, and the operating system's memory manager. None of that is specific to crashes. A symbolicator is just a sharp example of a common shape: data that's expensive to build and cheap to look up. Parse once, cache the right *shape*, let the kernel do the rest.++---++## The problem++Strip everything away and a symbolicator is one function:++```+Symbolicate(file, line, col) -> (origFile, origLine, origCol, fnName)+```++Calling it once is cheap. But exceptions are rarely a one-off; they come in bursts. You push a bug and suddenly you're getting spammed with the same crash, and the larger your app, the more of them come in. Client-side code is the worst case: a million browsers can hit the same crash in the same minute. Each error carries a dozen frames, spread across whatever bundles and releases those users happened to be running. So the real question isn't "how do I symbolicate a frame." I answered that already. It's "what sits behind `Symbolicate` so the ten-thousandth call in a sudden spike is as cheap as the first."++Let me build that up the way I actually built it.++---++## Stage 1: the dumb version that parses on every request++A request arrives carrying a stack frame: a file, a line, a column. To turn it back into source you need the build's debug artifact, and which one depends on the platform. JavaScript ships a source map plus the bundle. iOS ships a `.dSYM`. Dart ships a `.symbols` file. Different formats, one job: each maps a compiled position back to the original file, line, and name.++The simplest possible service does the whole thing inline, fresh on every request:++++Step 2 is where the cost is. Parsing the artifact walks the whole file and builds a sizable in-memory index: milliseconds of work, thousands of allocations. The lookup in step 3 is a plain search by comparison, effectively free.++Here's the waste. A release's debug artifact is frozen the moment you ship it; it never changes. Yet this version rebuilds the same index from the same bytes on every request, then throws it away. During a burst, a thousand frames a second off one broken release, you run that identical parse a thousand times a second and spend almost nothing on the lookups you actually came for.++So the fix names itself. Parse once per artifact, look up once per frame. That's the easy part. Making it hold up in production is the rest of the work.++---++## Stage 2: build the structure once, keep it in memory++Just how lopsided is that split? Here it is in Traceway, where the parse-and-resolve step is `BuildTW` and the per-frame lookup is `LookupTW`. A local `go test -bench` on a real pair, `preact.module.js` (10 KB) and its 69 KB source map:++```+BenchmarkBuildTW 975232 ns/op 771942 B/op 10260 allocs/op+BenchmarkLookupTW 44.24 ns/op 24 B/op 2 allocs/op+```++The lookup is **44 nanoseconds**. The parse in front of it is **975 microseconds**, three quarters of a megabyte of garbage, ten thousand allocations. The build is **~22,000× more expensive than the lookup it enables.** So you cache the parse and never repeat it. The interesting question is *what* to cache, and the whole post hinges on the answer.++The lazy answer is to keep the live parsed object: a Go struct full of slices and a `map[string]int32` name index. It works, but it's a sprawling object graph: thousands of little allocations, every one a pointer the garbage collector (GC) chases on every cycle, for a value that's immutable and that you'd like to keep around for hours. Cache ten thousand of those and you've turned a symbolicator into a GC benchmark.++So `BuildTW` doesn't return an object graph. It serializes to bytes, in Traceway's [`.tw` resolver format](https://docs.tracewayapp.com/symbolicator#the-tw-resolver-format): the compact, little-endian layout the engine compiles every build artifact into and runs on in production. A `.tw` is a cache artifact, never the source of truth, so it can afford to be dumb. A header, a sorted array of fixed-size **token** records (one per mapped position), then two interned string tables for file and function names:++++Every token is a fixed 24 bytes, six little-endian `uint32`s: the generated line and column (where it sits in the minified bundle), the original line and column (where it came from in your source), and two indices into the string tables (the source file and the function name). No pointers anywhere. A string is an *index*, and the tables turn an index into bytes. Here's the heart of the serializer:++```go title="backend/app/symbolicator/sourcemap/tw.go"+out = append(out, twMagic[:]...) // "TWSM"+out = binary.LittleEndian.AppendUint32(out, twVersion)+out = binary.LittleEndian.AppendUint32(out, uint32(len(b.tokens)))+// ... file + fn counts + a reserved word ...++for i := range b.tokens {+ t := &b.tokens[i]+ out = binary.LittleEndian.AppendUint32(out, t.genLine)+ out = binary.LittleEndian.AppendUint32(out, t.genCol)+ out = binary.LittleEndian.AppendUint32(out, t.srcLine)+ out = binary.LittleEndian.AppendUint32(out, t.srcCol)+ out = binary.LittleEndian.AppendUint32(out, uint32(t.fileIdx))+ out = binary.LittleEndian.AppendUint32(out, uint32(t.fnIdx))+}+```++The tokens are sorted by generated position. That's the whole point. A lookup becomes a **binary search over a slice of fixed-size records**, no decoding involved. `LookupTW` is just `sort.Search` plus a couple of field reads:++```go title="backend/app/symbolicator/sourcemap/tw.go"+idx := sort.Search(l.tokenCount, func(i int) bool {+ gl, gc := tokAt(i) // read token i's generated pos+ return gl > genLine || (gl == genLine && gc > genCol)+})+// idx-1 is the floor token: the greatest position <= the frame.+// read its srcLine/srcCol/fileIdx/fnIdx straight out of the bytes,+// resolve the two indices through the string tables, done.+```++That's the 44-nanosecond number. The two allocations it reports are the two strings it copies out at the end, the file name and the function name. Everything else is read in place. No parse, no garbage.++On the preact pair, `BuildTW` turns a 69 KB map and a 10 KB bundle into a **48 KB** `.tw` blob with about 2,000 tokens. That blob is one allocation the GC never looks inside. It's the unit we cache. The cache is a plain LRU keyed by file name, bounded by entry count *and* total bytes. (LRU means least-recently-used: when it fills up, whatever nobody has touched in the longest gets dropped.)++```go title="backend/app/symbolicator/twcache/mem.go"+func (s *twcachemem) get(name string) ([]byte, func(), bool) {+ s.mu.Lock()+ defer s.mu.Unlock()+ el, ok := s.items[name]+ if !ok {+ return nil, noop, false+ }+ s.order.MoveToFront(el) // mark most-recently-used+ return el.Value.(*memEntry).data, noop, true+}+```++The cache wrapper builds each file exactly once, even under a stampede: the first request for a file builds it while any concurrent requests for that file wait on the same in-flight load, instead of all calling `BuildTW` at once. The per-request path becomes:++```go+data, done, err := cache.Get(ctx, fileKey, func(ctx context.Context) ([]byte, error) {+ return sourcemap.BuildTW(mapBytes, bundleBytes) // runs once per file+})+defer done()+frame, ok := sourcemap.LookupTW(data, line, col) // 44 ns, every frame after the first+```++++Now the first frame from a bundle pays the 975 µs. Every frame after it pays 44 ns, for the rest of that bundle's life in cache. At a thousand errors a second over a working set that fits in memory, we just deleted nearly all the CPU.++Why serialize to bytes instead of caching the parsed struct? Three reasons, and the third is the one that matters:++1. **It's compact.** 48 KB beats a sprawl of thousands of small objects.+2. **It's invisible to the GC.** One `[]byte` with no inner pointers. The collector skips straight over it, however many thousands we hold.+3. **It's already the exact shape you'd write to a file.** A flat, pointer-free, offset-addressed byte array is already a file format. That's the entire next stage.++Hold that third point. But first, the catch that ends Stage 2: this cache lives entirely in RAM.++To keep every lookup fast, every `.tw` you might need has to be resident at once. The whole working set has to fit in memory. For one small app it does, a handful of bundles. A platform symbolicating at scale is the opposite. A mobile app has a long tail of versions still installed on real devices. A website ships a new bundle on every deploy, and crashes keep arriving from every release a user is still running. That's thousands of distinct artifacts, each with its own `.tw`, and the old ones never fully go quiet. Hold them all in memory, on every replica, and you're paying a bill you can't afford. The LRU's only answer is to evict, which just turns the overflow back into 975 µs rebuilds.++Two smaller cracks sit next to it. A restart is a cold start: the cache is gone, and the first wave of every bundle re-parses from scratch. Replicas don't share: ten servers behind a load balancer keep ten separate caches, each re-parsing the same bundle the first time a frame lands there.++Three problems, one root. The structure lives on one heap, in one process, for only as long as that process runs. The next stage moves it off the heap entirely, and fixes all three at once.++---++## Stage 3: store it and mmap it++Reread reason 3 above. The thing we cache is a flat `[]byte` with no pointers; every "reference" is just an integer offset from the start of the buffer. A structure like that doesn't care whether its bytes live in a heap allocation or in a file on disk. The bytes *are* the structure.++So persisting it is `os.WriteFile`. And loading it is `mmap`, the system call that maps a file straight into your process's memory. You get back a byte array that *is* the file on disk. The OS pulls in each chunk the first time you touch it, a **page fault**, instead of copying the whole thing up front:++```go title="backend/app/symbolicator/twcache/mmap_unix.go"+func mmapFile(path string) ([]byte, func(), error) {+ f, err := os.Open(path)+ if err != nil {+ return nil, nil, err+ }+ defer f.Close()++ st, _ := f.Stat()+ size := st.Size()++ data, err := syscall.Mmap(int(f.Fd()), 0, int(size),+ syscall.PROT_READ, syscall.MAP_SHARED)+ if err != nil {+ return nil, nil, err+ }+ return data, func() { _ = syscall.Munmap(data) }, nil+}+```++Now look at the signature we've been calling all along:++```go+func LookupTW(data []byte, genLine, genCol uint32) (StackTraceFrame, bool)+```++It takes a `[]byte`. **It never cared where the bytes came from.** A heap buffer from `make`, a memory-mapped file from `syscall.Mmap`: to `LookupTW` they're the same thing. We designed the structure so they could be. Stage 2 and Stage 3 run the identical lookup code over identical bytes. The only thing that changed is who owns the pages.++And that's what the binary search buys us: we search the file **in place**. No read, no decode, no copy into a struct. The search over ~2,000 tokens touches about eleven 24-byte records. On a 48 KB file those land in a few pages. So the first lookup costs a few page faults, and every one after it is a plain memory read at RAM speed. The disk store wires `mmapFile` straight into the cache's `get`:++```go title="backend/app/symbolicator/twcache/disk.go"+func (s *twcachedisk) get(name string) ([]byte, func(), bool) {+ path, err := s.path(name)+ if err != nil {+ return nil, noop, false+ }+ data, unmap, err := mmapFile(path) // []byte backed by the file+ if err != nil {+ return nil, noop, false+ }+ s.noteUse(name, int64(len(data)))+ return data, unmap, true // caller calls unmap when done+}+```++That `done`/`unmap` is the one bit of discipline mmap adds. The `[]byte` is only valid until you unmap it, so the request borrows it, does its lookups, and releases it at the end. It's safe because `LookupTW` *copies out* the two strings it returns: the file name and function name become real Go strings on the heap, so the resolved frame outlives the mapping even though everything else was read straight from the file's pages.++Here's what quietly falls out of this, the payoff for all three of Stage 2's leftover problems:++- **The OS page cache becomes the cache tier, for free.** The page cache is just the RAM the OS already uses to hold recently-read file data. Hot `.tw` files stay resident there, because the kernel keeps touched pages around. Cold ones get dropped under memory pressure by the kernel's own reclaim. I handed the eviction problem to the operating system, which has tuned that exact algorithm for fifty years.+- **It survives restarts.** The `.tw` files are on disk. A restart re-`mmap`s them in microseconds. No re-parse, no cold-start stampede.+- **Replicas share.** Put the prebuilt `.tw` blobs in shared object storage. Every replica pulls the same 48 KB artifact instead of re-deriving it from 79 KB of map-plus-bundle. A fresh server is warm almost immediately.++In production these stack into one cascade, cheapest first:++1. **Local mmap cache.** The `.tw` file is on this box. Map it, search it, done in microseconds.+2. **Object storage.** Not local yet. Download the prebuilt 48 KB `.tw`, write it down, map it. Still no parse.+3. **Rebuild.** No `.tw` exists anywhere. This is the first crash of a brand-new release. Run the full 975 µs `BuildTW` *once*, then write the result to storage so steps 1 and 2 cover every replica after that.++++The expensive path, Stage 1's full parse, is now the rare cold-cold case. It runs once per release, ever, instead of once per frame.++---++## The shape of the win++The same `LookupTW`, fed from four different places:++| Stage | Per-frame cost | Survives restart | Shared across replicas | GC pressure |+|---|---|---|---|---|+| 1 · parse every request | ~975 µs | no | no | 10k allocs/frame |+| 2 · in-memory `.tw` cache | ~44 ns (after first) | no | no | one `[]byte`, no scan |+| 3 · disk + mmap | ~44 ns + a few page faults cold | yes | yes (via storage) | none (pages aren't heap) |++What carried the whole thing was a data-layout choice from Stage 2, not a clever caching layer. We made the searchable structure a flat, pointer-free `[]byte` instead of a Go object graph. That one choice collapsed two problems into one: caching it in memory and storing it on disk became the same bytes handed to the same function. And the kernel's page cache, which I didn't write and don't tune, does my eviction.++---++## Does it survive contact with production?++A 44-nanosecond lookup is a nice number on a bench. The real question is what happens under a firehose: thousands of stack traces a second, hundreds of releases, a working set bigger than RAM. So I ran the repo's standard processor benchmark against the best baseline I could find.++The baseline is **Honeycomb's open-source OpenTelemetry symbolicator processors**, the `sourcemapprocessor` for JavaScript and the `dsymprocessor` for iOS. Under the hood they wrap **Sentry's `symbolic`**, the Rust library that powers Sentry's own symbolication. It's the engine plenty of production error tracking already runs on, and it's mature, well-optimized Rust. The only way a Go service beats that is by doing less work, not faster work, and that's the whole design: on the hot path, skip the parse entirely.++To compare the two symbolicators and nothing else, I run both as OpenTelemetry Collector processors, the plugin slot that rewrites telemetry as it passes through. Honeycomb's are the `sourcemapprocessor` and `dsymprocessor`; Traceway's is its own. Both sit in the same collector, behind the same Go load generator firing crash reports over OTLP, in front of the same small Rust receiver that checks every frame came back resolved, on the same Hetzner box (a ccx33: 8 vCPU, 32 GB). Swap the processor, change nothing else. Load ramps from 8 to 256 connections, and I keep the peak.++Memory needs one caveat, because it's where the gap is widest. RAM is sampled from *outside* the process, since Go's own heap stats can't see Honeycomb's parsed maps (they live on the C heap inside Sentry's `symbolic`) and would over-count Traceway's mmap'd pages (the kernel reclaims those on demand). The corpus is a real minified bundle padded to a production-realistic size, so a cache miss costs what it would in production. Then three scenarios, each pointed at a different failure mode:++- **hot**: one bundle, always cache-warm. The ceiling: pure lookup throughput with zero misses.+- **churn**: 512 bundles cycling through a 128-entry cache, so almost every request misses. This is the real world, where traffic spans every release you've ever shipped, and it's where the cost of a miss gets exposed.+- **oom**: thousands of fat bundles on a small 2-vCPU, 8 GB box, a working set far bigger than RAM. The survival test: who stays up, and who the kernel kills. (The padding grows Traceway's token table too, so nothing's tilted in its favor.)++Four columns carry the story. `max stacks/s` is throughput, stack traces resolved per second. `p99 ms` is tail latency, the slowest 1% of requests, the part users feel. `peak RSS MB` is the most RAM the process ever held. `symb%` is the correctness check, the fraction of frames that came back resolved. `outcome` is whether the process lived or got killed.++Here's JavaScript, Traceway against Honeycomb:++| impl | scenario | max stacks/s | p99 ms | peak RSS MB | symb% | outcome |+|---|---|---|---|---|---|---|+| honeycomb | hot | 20,735 | 373 | 125 | 100 | survived |+| honeycomb | churn | 938 | 2,962 | 4,491 | 100 | survived |+| honeycomb | oom | 938 | 2,962 | 6,844 | 99 | survived |+| traceway-oxc-disk | hot | 30,910 | 4.1 | 379 | 100 | survived |+| traceway-oxc-disk | churn | 29,923 | 17.7 | 361 | 100 | survived |+| traceway-oxc-disk | oom | 29,923 | 17.7 | 6,851 | 100 | survived |++Correctness is a tie. Everything resolves ~100% of frames. That's the price of admission, not a differentiator. The differentiator is everything else:++- **Churn throughput: 938 → 29,923 stacks/s, about 32×.** This is the whole post in one number. On every cache miss Honeycomb re-parses the raw source map through `symbolic`, the Stage 1 "dumb version" tax paid forever, and it pins about one core: its average CPU sat near 104%, roughly a single core busy, while Traceway spread across seven cores at ~690%. Traceway on a miss just re-opens a precompiled `.tw` file and maps it. The 32× *is* the distance between "re-derive the structure" and "the structure is already the file."+- **Latency: hot p99 373 ms → 4.1 ms (~90×), churn p99 2,962 ms → 17.7 ms (~168×).** When the cache is warm both are quick-ish per core. But at saturation Honeycomb's tail blows out to multiple seconds. A symbolicator with a 3-second p99 is why your errors show up late.+- **Memory under churn: 4,491 MB → 361 MB, about 12× less.** Honeycomb keeps the raw map JSON and the minified bundle resident per cache entry. Traceway throws both away the instant it has compiled the compact `.tw`. (Both hit ~6.8 GB peak in the oom run, but read that with the asterisk above. Traceway's are reclaimable page-cache pages, and it sustains 30k/s there where Honeycomb manages 938. Survival is the next section.)++---++## The mode I run in production: oxc + disk++Those Traceway rows were one configuration. The benchmark swept four, two parsers crossed with two cache tiers. The full matrix tells you *which* knobs to turn:++| impl | scenario | max stacks/s | p99 ms | peak RSS MB | outcome |+|---|---|---|---|---|---|+| traceway-oxc-disk | churn | 29,923 | 17.7 | 361 | survived |+| traceway-oxc-disk | oom | 29,923 | 17.7 | 6,851 | survived |+| traceway-oxc-mem | churn | 16,755 | 128.8 | 584 | survived |+| traceway-oxc-mem | oom | 16,755 | 128.8 | 7,469 | **died@40s** |+| traceway-goja-disk | churn | 29,804 | 22.7 | 337 | survived |+| traceway-goja-mem | churn | 3,414 | 6,342 | 1,309 | survived |+| traceway-goja-mem | oom | 3,414 | 6,342 | 7,456 | **died@236s** |++Two independent decisions fall out, and the data picks both.++**Disk over memory, because memory dies.** This is the Stage 2 → Stage 3 jump, now with a body count. Both `-mem` variants get OOM-killed on the 8 GB box (`oxc-mem` at 40 seconds, `goja-mem` at 236). The in-memory cache holds every `.tw` on the Go heap. The heap can't be reclaimed under pressure, so the kernel kills the process. Both `-disk` variants *survive* the same workload. An mmap'd `.tw` is just file pages the kernel can drop and re-fault on demand. That's eviction I don't write and can't get wrong. Disk also keeps churn memory flat, 361 MB against `goja-mem`'s 1,309 MB, because the working set lives in the page cache, not the heap.++**oxc over goja, because the parser is the miss cost.** The bundle parser only runs on a cache *miss* (the `BuildTW` parse). So it's invisible in `hot` and decisive in `churn`. `goja` is a pure-Go JavaScript parser: zero dependencies, always available, and what the stock build ships. `oxc` is a Rust parser, the one `rspack` and friends use, linked in over cgo (Go calling a native library). On the churn run `oxc-disk` holds a 17.7 ms p99 against `goja-disk`'s 22.7 ms, and the gap widens the harder you lean on misses. Pay the build complexity and the cold path gets cheaper.++Put them together. **oxc + disk** has the best latency, and it's the only corner that stays flat on memory *and* survives the OOM box. That's what I run. It's opt-in, though. The stock image ships `goja` + `memory` with cgo off, because it needs no Rust toolchain and is plenty for a single small project. To switch to the production mode, build the oxc shim in once and flip two env vars:++```bash title="production symbolicator config"+# one-time: compile the Rust parser shim, then build with cgo + the oxc tag+# scripts/build-oxc-shim.sh && go build -tags oxc ./...++SYMBOLICATOR_PARSER=oxc+SOURCEMAP_CACHE_TYPE=disk+SOURCEMAP_DISK_CACHE_PATH=/var/lib/traceway/twcache+SOURCEMAP_DISK_CACHE_MAX_MB=2048+```++---++## The same trick, every language++Nothing in Stage 2 or Stage 3 was about JavaScript. The `.tw` format is a sorted token table plus two string tables. It doesn't care whether those tokens came from a JavaScript source map, a Dart `.symbols` file, or an iOS `.dSYM`. Those last two are the debug files mobile platforms ship instead of source maps, both built on DWARF (the same debug format a native debugger reads). Every one of them gets *flattened* into the same `.tw` on a cache miss, and from there it's the same binary search over the same mmap'd bytes. So the disk tier (survival, flat memory, a cache that outlives a restart) comes for free the moment you teach the symbolicator a new format.++The benchmark bears that out. Dart first (no Honeycomb column, since their processor doesn't support Dart or Flutter):++| impl | scenario | max stacks/s | p99 ms | peak RSS MB | outcome |+|---|---|---|---|---|---|+| traceway-dart-disk | hot | 24,304 | 425.9 | 137 | survived |+| traceway-dart-disk | churn | 22,915 | 398.5 | 138 | survived |+| traceway-dart-mem | hot | 28,605 | 5.5 | 361 | survived |+| traceway-dart-mem | churn | 8,586 | 72.7 | 529 | survived |++The same pattern holds. `disk` wins churn throughput 2.7× and holds memory at 138 MB against `mem`'s 529 MB (and 2,165 MB on the oom soak). `mem` is much faster on the always-warm `hot` path (a 5.5 ms p99, where disk's tail runs far longer), but `hot` is the one scenario that never happens at scale. The flatten-once, mmap-forever pattern does the identical job for DWARF that it did for source maps.++And iOS, head-to-head with Honeycomb's `dsymprocessor`:++| impl | scenario | max stacks/s | p99 ms | peak RSS MB | outcome |+|---|---|---|---|---|---|+| honeycomb-ios | churn | 11,433 | 285 | 9,727 | survived |+| traceway-ios-disk | churn | 46,174 | 303 | 94 | survived |+| traceway-ios-mem | churn | 29,275 | 13.3 | 275 | survived |++`traceway-ios-disk` pushes ~4× the throughput of `honeycomb-ios` at **94 MB against Honeycomb's 9,727 MB, about 100× less memory.** Honeycomb keeps each parsed `symbolic` archive resident. Traceway keeps a compact `.tw` and lets the kernel page it. Different binary format, same win. I didn't write a line of new cache code to get it.++---++## Conclusion++A symbolicator is the least glamorous corner of an observability platform, and I sank weeks into it anyway, decoding source maps and DWARF by hand. "Close enough" symbolication is what fails you at 3 a.m., when a frame says `n` and you need `validateCheckout`. Getting that boring layer right is why I'm building Traceway from the ground up instead of renting it.++The part I'd underrated going in was `mmap`. I'd read about it for years and never reached for it in production until this one. You hand the kernel a file, get back a byte array that *is* the file, and the OS page cache does the eviction for you. The fallback, `os.ReadFile`, just copies every byte onto the heap for the garbage collector to chase. Handing the cache to the operating system, instead of writing a worse one, is the thing I'll reach for first next time.++The format is done and it's what ships today, but one idea keeps tugging at me for a future version, just because it'd be fun to chase. Every token is a full 24 bytes of absolute values, so a lookup scatters across the file even though `mmap` already hands back a whole 4 to 16 KB page each time it faults. Frame-of-reference encoding could tighten that: group the tokens into fixed-length frames, keep one base per frame, and store each token as a tiny delta from it. The file would shrink, and a lookup would stay inside a single frame, one page fault instead of a dozen. I have no idea yet how much it actually buys, which is exactly the kind of thing I'd love to find out.++---++*Traceway is MIT-licensed and OpenTelemetry-native, and the symbolicator above ships with it: the `.tw` format, the mmap cache, all of it. The algorithm half of this story is in the [source map deep dive](/blog/deep-dive-into-the-sourcemaps) and the [Flutter one](/blog/flutter-symbolication). This was the systems half. Ever made something faster on disk than in memory because the on-disk shape was the right shape? I'd love to hear it. Reach me at [email protected].*diff --git a/website/public/blog/system-design-of-a-symbolicator/cache-in-memory.svg b/website/public/blog/system-design-of-a-symbolicator/cache-in-memory.svgnew file mode 100644index 00000000..d1d315f0--- /dev/null+++ b/website/public/blog/system-design-of-a-symbolicator/cache-in-memory.svg@@ -0,0 +1,94 @@+<?xml version="1.0" encoding="UTF-8" standalone="no"?>+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">+<!-- Generated by graphviz version 15.0.0 (20260523.1842)+ -->+<!-- Title: cache_in_memory Pages: 1 -->+<svg width="445pt" height="551pt"+ viewBox="0.00 0.00 445.00 551.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(14.4 536.92)">+<title>cache_in_memory</title>+<!-- req -->+<g id="node1" class="node">+<title>req</title>+<path fill="#0f131c" stroke="#283041" stroke-width="1.3" d="M225.07,-522.52C225.07,-522.52 65.62,-522.52 65.62,-522.52 59.62,-522.52 53.62,-516.52 53.62,-510.52 53.62,-510.52 53.62,-482.6 53.62,-482.6 53.62,-476.6 59.62,-470.6 65.62,-470.6 65.62,-470.6 225.07,-470.6 225.07,-470.6 231.07,-470.6 237.07,-476.6 237.07,-482.6 237.07,-482.6 237.07,-510.52 237.07,-510.52 237.07,-516.52 231.07,-522.52 225.07,-522.52"/>+<text xml:space="preserve" text-anchor="start" x="75.22" y="-497.26" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">incoming stack frame</text>+<text xml:space="preserve" text-anchor="start" x="81.6" y="-485.16" font-family="Menlo" font-size="12.00" fill="#8a93a6">app.min.js:1:5114</text>+</g>+<!-- cache -->+<g id="node2" class="node">+<title>cache</title>+<path fill="#08161c" stroke="#00d4ff" stroke-width="1.3" d="M243.45,-437.6C243.45,-437.6 47.25,-437.6 47.25,-437.6 41.25,-437.6 35.25,-431.6 35.25,-425.6 35.25,-425.6 35.25,-397.68 35.25,-397.68 35.25,-391.68 41.25,-385.68 47.25,-385.68 47.25,-385.68 243.45,-385.68 243.45,-385.68 249.45,-385.68 255.45,-391.68 255.45,-397.68 255.45,-397.68 255.45,-425.6 255.45,-425.6 255.45,-431.6 249.45,-437.6 243.45,-437.6"/>+<text xml:space="preserve" text-anchor="start" x="56.85" y="-412.34" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">look up the file in the cache</text>+<text xml:space="preserve" text-anchor="start" x="65.85" y="-400.24" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#8fe6ff">in-memory, LRU, keyed by file</text>+</g>+<!-- req->cache -->+<g id="edge1" class="edge">+<title>req->cache</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M145.35,-470.23C145.35,-463.38 145.35,-455.84 145.35,-448.53"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="148.33,-448.72 145.35,-440.22 142.38,-448.72 148.33,-448.72"/>+</g>+<!-- build -->+<g id="node3" class="node">+<title>build</title>+<path fill="#1b1115" stroke="#ff5a5f" stroke-width="1.3" d="M243.07,-341.68C243.07,-341.68 47.63,-341.68 47.63,-341.68 41.62,-341.68 35.62,-335.68 35.62,-329.68 35.62,-329.68 35.62,-301.76 35.62,-301.76 35.62,-295.76 41.62,-289.76 47.62,-289.76 47.62,-289.76 243.07,-289.76 243.07,-289.76 249.07,-289.76 255.07,-295.76 255.07,-301.76 255.07,-301.76 255.07,-329.68 255.07,-329.68 255.07,-335.68 249.07,-341.68 243.07,-341.68"/>+<text xml:space="preserve" text-anchor="start" x="57.22" y="-316.42" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">parse → build the .tw index</text>+<text xml:space="preserve" text-anchor="start" x="75.22" y="-304.32" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#ff9a9d">slow, but only once per file</text>+</g>+<!-- cache->build -->+<g id="edge2" class="edge">+<title>cache->build</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M144.99,-385.08C144.93,-379.98 144.88,-374.66 144.85,-369.68 144.82,-364.25 144.83,-358.5 144.86,-352.87"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="147.83,-352.95 144.93,-344.43 141.88,-352.9 147.83,-352.95"/>+<text xml:space="preserve" text-anchor="middle" x="210.1" y="-359.23" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#8a93a6">  miss (first time for this file)</text>+</g>+<!-- look -->+<g id="node5" class="node">+<title>look</title>+<path fill="#0c1714" stroke="#22e0a8" stroke-width="1.3" d="M233.32,-136.84C233.32,-136.84 57.37,-136.84 57.37,-136.84 51.38,-136.84 45.38,-130.84 45.38,-124.84 45.38,-124.84 45.38,-96.92 45.38,-96.92 45.38,-90.92 51.38,-84.92 57.38,-84.92 57.38,-84.92 233.32,-84.92 233.32,-84.92 239.32,-84.92 245.32,-90.92 245.32,-96.92 245.32,-96.92 245.32,-124.84 245.32,-124.84 245.32,-130.84 239.32,-136.84 233.32,-136.84"/>+<text xml:space="preserve" text-anchor="start" x="66.97" y="-111.58" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">search the cached index</text>+<text xml:space="preserve" text-anchor="start" x="87.22" y="-99.48" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#7ff0c8">fast: 44 ns, no parsing</text>+</g>+<!-- cache->look -->+<g id="edge3" class="edge">+<title>cache:e->look:e</title>+<path fill="none" stroke="#22e0a8" stroke-width="1.4" d="M256.45,-411.64C386.67,-411.64 380.51,-126.53 256.65,-111.5"/>+<polygon fill="#22e0a8" stroke="#22e0a8" stroke-width="1.4" points="257.1,-108.54 248.44,-111.01 256.75,-114.48 257.1,-108.54"/>+<text xml:space="preserve" text-anchor="start" x="351.84" y="-263.31" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#7ff0c8">hit</text>+<text xml:space="preserve" text-anchor="start" x="351.84" y="-251.31" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#7ff0c8">(almost every</text>+<text xml:space="preserve" text-anchor="start" x="351.84" y="-239.31" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#7ff0c8">request)</text>+</g>+<!-- store -->+<g id="node4" class="node">+<title>store</title>+<path fill="#08161c" stroke="#00d4ff" stroke-width="1.3" d="M234.82,-221.76C234.82,-221.76 55.87,-221.76 55.87,-221.76 49.87,-221.76 43.88,-215.76 43.88,-209.76 43.88,-209.76 43.88,-181.84 43.88,-181.84 43.88,-175.84 49.88,-169.84 55.88,-169.84 55.88,-169.84 234.82,-169.84 234.82,-169.84 240.82,-169.84 246.82,-175.84 246.82,-181.84 246.82,-181.84 246.82,-209.76 246.82,-209.76 246.82,-215.76 240.82,-221.76 234.82,-221.76"/>+<text xml:space="preserve" text-anchor="start" x="65.47" y="-196.5" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">keep the .tw in the cache</text>+<text xml:space="preserve" text-anchor="start" x="68.47" y="-184.4" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#8fe6ff">reused by every later request</text>+</g>+<!-- build->store -->+<g id="edge4" class="edge">+<title>build->store</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M145.35,-289.13C145.35,-272.68 145.35,-251.05 145.35,-232.86"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="148.33,-232.96 145.35,-224.46 142.38,-232.96 148.33,-232.96"/>+</g>+<!-- store->look -->+<g id="edge5" class="edge">+<title>store->look</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M145.35,-169.47C145.35,-162.62 145.35,-155.08 145.35,-147.77"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="148.33,-147.96 145.35,-139.46 142.38,-147.96 148.33,-147.96"/>+</g>+<!-- out -->+<g id="node6" class="node">+<title>out</title>+<path fill="#15121f" stroke="#7c5cff" stroke-width="1.3" d="M278.7,-51.92C278.7,-51.92 12,-51.92 12,-51.92 6,-51.92 0,-45.92 0,-39.92 0,-39.92 0,-12 0,-12 0,-6 6,0 12,0 12,0 278.7,0 278.7,0 284.7,0 290.7,-6 290.7,-12 290.7,-12 290.7,-39.92 290.7,-39.92 290.7,-45.92 284.7,-51.92 278.7,-51.92"/>+<text xml:space="preserve" text-anchor="start" x="96.97" y="-26.66" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">resolved frame</text>+<text xml:space="preserve" text-anchor="start" x="21.6" y="-14.56" font-family="Menlo" font-size="12.00" fill="#b9a7ff">checkout.js:48 · validateCheckout</text>+</g>+<!-- look->out -->+<g id="edge6" class="edge">+<title>look->out</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M145.35,-84.55C145.35,-77.7 145.35,-70.16 145.35,-62.85"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="148.33,-63.04 145.35,-54.54 142.38,-63.04 148.33,-63.04"/>+</g>+</g>+</svg>diff --git a/website/public/blog/system-design-of-a-symbolicator/parse-on-every-request.svg b/website/public/blog/system-design-of-a-symbolicator/parse-on-every-request.svgnew file mode 100644index 00000000..5c821456--- /dev/null+++ b/website/public/blog/system-design-of-a-symbolicator/parse-on-every-request.svg@@ -0,0 +1,92 @@+<?xml version="1.0" encoding="UTF-8" standalone="no"?>+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">+<!-- Generated by graphviz version 15.0.0 (20260523.1842)+ -->+<!-- Title: parse_every_request Pages: 1 -->+<svg width="467pt" height="517pt"+ viewBox="0.00 0.00 467.00 517.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(14.4 502.67)">+<title>parse_every_request</title>+<!-- req -->+<g id="node1" class="node">+<title>req</title>+<path fill="#0f131c" stroke="#283041" stroke-width="1.3" d="M235.95,-488.27C235.95,-488.27 76.5,-488.27 76.5,-488.27 70.5,-488.27 64.5,-482.27 64.5,-476.27 64.5,-476.27 64.5,-448.35 64.5,-448.35 64.5,-442.35 70.5,-436.35 76.5,-436.35 76.5,-436.35 235.95,-436.35 235.95,-436.35 241.95,-436.35 247.95,-442.35 247.95,-448.35 247.95,-448.35 247.95,-476.27 247.95,-476.27 247.95,-482.27 241.95,-488.27 235.95,-488.27"/>+<text xml:space="preserve" text-anchor="start" x="86.1" y="-463.01" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">incoming stack frame</text>+<text xml:space="preserve" text-anchor="start" x="92.48" y="-450.91" font-family="Menlo" font-size="12.00" fill="#8a93a6">app.min.js:1:5114</text>+</g>+<!-- load -->+<g id="node2" class="node">+<title>load</title>+<path fill="#151a24" stroke="#283041" stroke-width="1.3" d="M272.33,-405.35C272.33,-405.35 40.12,-405.35 40.12,-405.35 34.13,-405.35 28.13,-399.35 28.13,-393.35 28.13,-393.35 28.13,-365.43 28.13,-365.43 28.13,-359.43 34.13,-353.43 40.13,-353.43 40.13,-353.43 272.33,-353.43 272.33,-353.43 278.33,-353.43 284.33,-359.43 284.33,-365.43 284.33,-365.43 284.33,-393.35 284.33,-393.35 284.33,-399.35 278.33,-405.35 272.33,-405.35"/>+<text xml:space="preserve" text-anchor="start" x="74.48" y="-380.09" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">1. load the debug artifact</text>+<text xml:space="preserve" text-anchor="start" x="49.73" y="-367.99" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#8a93a6">source map + bundle · .dSYM · .symbols</text>+</g>+<!-- req->load -->+<g id="edge1" class="edge">+<title>req->load</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M156.23,-435.71C156.23,-429.45 156.23,-422.64 156.23,-416"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="159.2,-416.45 156.23,-407.95 153.25,-416.45 159.2,-416.45"/>+</g>+<!-- parse -->+<g id="node3" class="node">+<title>parse</title>+<path fill="#1b1115" stroke="#ff5a5f" stroke-width="1.3" d="M300.45,-322.43C300.45,-322.43 12,-322.43 12,-322.43 6,-322.43 0,-316.43 0,-310.43 0,-310.43 0,-282.51 0,-282.51 0,-276.51 6,-270.51 12,-270.51 12,-270.51 300.45,-270.51 300.45,-270.51 306.45,-270.51 312.45,-276.51 312.45,-282.51 312.45,-282.51 312.45,-310.43 312.45,-310.43 312.45,-316.43 306.45,-322.43 300.45,-322.43"/>+<text xml:space="preserve" text-anchor="start" x="52.73" y="-297.17" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">2. parse → build a lookup index</text>+<text xml:space="preserve" text-anchor="start" x="21.6" y="-285.07" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#ff9a9d">slow: reads the whole file, thousands of allocations</text>+</g>+<!-- load->parse -->+<g id="edge2" class="edge">+<title>load->parse</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M156.23,-352.79C156.23,-346.53 156.23,-339.72 156.23,-333.08"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="159.2,-333.53 156.23,-325.03 153.25,-333.53 159.2,-333.53"/>+</g>+<!-- search -->+<g id="node4" class="node">+<title>search</title>+<path fill="#0c1714" stroke="#22e0a8" stroke-width="1.3" d="M271.95,-206.76C271.95,-206.76 40.5,-206.76 40.5,-206.76 34.5,-206.76 28.5,-200.76 28.5,-194.76 28.5,-194.76 28.5,-166.84 28.5,-166.84 28.5,-160.84 34.5,-154.84 40.5,-154.84 40.5,-154.84 271.95,-154.84 271.95,-154.84 277.95,-154.84 283.95,-160.84 283.95,-166.84 283.95,-166.84 283.95,-194.76 283.95,-194.76 283.95,-200.76 277.95,-206.76 271.95,-206.76"/>+<text xml:space="preserve" text-anchor="start" x="50.1" y="-181.5" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">3. search the index for the frame</text>+<text xml:space="preserve" text-anchor="start" x="103.35" y="-169.4" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#7ff0c8">fast: a single lookup</text>+</g>+<!-- parse->search -->+<g id="edge3" class="edge">+<title>parse->search</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M156.23,-269.96C156.23,-254.46 156.23,-234.43 156.23,-217.39"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="159.2,-217.84 156.23,-209.34 153.25,-217.84 159.2,-217.84"/>+</g>+<!-- ret -->+<g id="node5" class="node">+<title>ret</title>+<path fill="#151a24" stroke="#283041" stroke-width="1.3" d="M293.33,-123.84C293.33,-123.84 19.13,-123.84 19.13,-123.84 13.13,-123.84 7.13,-117.84 7.13,-111.84 7.13,-111.84 7.13,-94.92 7.13,-94.92 7.13,-88.92 13.13,-82.92 19.13,-82.92 19.13,-82.92 293.33,-82.92 293.33,-82.92 299.33,-82.92 305.33,-88.92 305.33,-94.92 305.33,-94.92 305.33,-111.84 305.33,-111.84 305.33,-117.84 299.33,-123.84 293.33,-123.84"/>+<text xml:space="preserve" text-anchor="start" x="28.73" y="-98.58" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">4. return the result, then drop the index</text>+</g>+<!-- search->ret -->+<g id="edge4" class="edge">+<title>search->ret</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M156.23,-154.27C156.23,-148.02 156.23,-141.28 156.23,-134.84"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="159.2,-135.02 156.23,-126.52 153.25,-135.02 159.2,-135.02"/>+</g>+<!-- ret->load -->+<g id="edge6" class="edge">+<title>ret:e->load:e</title>+<path fill="none" stroke="#ff5a5f" stroke-width="1.4" stroke-dasharray="5,2" d="M306.33,-103.38C349.65,-103.38 410.99,-363.98 295.85,-378.74"/>+<polygon fill="#ff5a5f" stroke="#ff5a5f" stroke-width="1.4" points="295.74,-375.76 287.44,-379.26 296.11,-381.7 295.74,-375.76"/>+<text xml:space="preserve" text-anchor="start" x="361.75" y="-246.01" font-family="Helvetica,sans-Serif" font-size="10.00" fill="#ff8a8d">  next request:</text>+<text xml:space="preserve" text-anchor="start" x="361.75" y="-234.76" font-family="Helvetica,sans-Serif" font-size="10.00" fill="#ff8a8d">  nothing kept,</text>+<text xml:space="preserve" text-anchor="start" x="361.75" y="-223.51" font-family="Helvetica,sans-Serif" font-size="10.00" fill="#ff8a8d">  reload + reparse</text>+</g>+<!-- out -->+<g id="node6" class="node">+<title>out</title>+<path fill="#15121f" stroke="#7c5cff" stroke-width="1.3" d="M289.58,-51.92C289.58,-51.92 22.87,-51.92 22.87,-51.92 16.87,-51.92 10.88,-45.92 10.88,-39.92 10.88,-39.92 10.88,-12 10.88,-12 10.88,-6 16.88,0 22.88,0 22.88,0 289.58,0 289.58,0 295.58,0 301.58,-6 301.58,-12 301.58,-12 301.58,-39.92 301.58,-39.92 301.58,-45.92 295.58,-51.92 289.58,-51.92"/>+<text xml:space="preserve" text-anchor="start" x="107.85" y="-26.66" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">resolved frame</text>+<text xml:space="preserve" text-anchor="start" x="32.48" y="-14.56" font-family="Menlo" font-size="12.00" fill="#b9a7ff">checkout.js:48 · validateCheckout</text>+</g>+<!-- ret->out -->+<g id="edge5" class="edge">+<title>ret->out</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M156.23,-82.56C156.23,-76.49 156.23,-69.61 156.23,-62.82"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="159.2,-62.95 156.23,-54.45 153.25,-62.95 159.2,-62.95"/>+</g>+</g>+</svg>diff --git a/website/public/blog/system-design-of-a-symbolicator/store-and-mmap.svg b/website/public/blog/system-design-of-a-symbolicator/store-and-mmap.svgnew file mode 100644index 00000000..9720584b--- /dev/null+++ b/website/public/blog/system-design-of-a-symbolicator/store-and-mmap.svg@@ -0,0 +1,126 @@+<?xml version="1.0" encoding="UTF-8" standalone="no"?>+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">+<!-- Generated by graphviz version 15.0.0 (20260523.1842)+ -->+<!-- Title: store_and_mmap Pages: 1 -->+<svg width="493pt" height="663pt"+ viewBox="0.00 0.00 493.00 663.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(14.4 648.76)">+<title>store_and_mmap</title>+<!-- req -->+<g id="node1" class="node">+<title>req</title>+<path fill="#0f131c" stroke="#283041" stroke-width="1.3" d="M413.33,-634.36C413.33,-634.36 253.88,-634.36 253.88,-634.36 247.88,-634.36 241.88,-628.36 241.88,-622.36 241.88,-622.36 241.88,-594.44 241.88,-594.44 241.88,-588.44 247.88,-582.44 253.88,-582.44 253.88,-582.44 413.33,-582.44 413.33,-582.44 419.33,-582.44 425.33,-588.44 425.33,-594.44 425.33,-594.44 425.33,-622.36 425.33,-622.36 425.33,-628.36 419.33,-634.36 413.33,-634.36"/>+<text xml:space="preserve" text-anchor="start" x="263.48" y="-609.1" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">incoming stack frame</text>+<text xml:space="preserve" text-anchor="start" x="269.85" y="-597" font-family="Menlo" font-size="12.00" fill="#8a93a6">app.min.js:1:5114</text>+</g>+<!-- local -->+<g id="node2" class="node">+<title>local</title>+<path fill="#151a24" stroke="#283041" stroke-width="1.3" d="M409.58,-549.44C409.58,-549.44 257.62,-549.44 257.62,-549.44 251.63,-549.44 245.63,-543.44 245.63,-537.44 245.63,-537.44 245.63,-509.52 245.63,-509.52 245.63,-503.52 251.63,-497.52 257.62,-497.52 257.62,-497.52 409.58,-497.52 409.58,-497.52 415.58,-497.52 421.58,-503.52 421.58,-509.52 421.58,-509.52 421.58,-537.44 421.58,-537.44 421.58,-543.44 415.58,-549.44 409.58,-549.44"/>+<text xml:space="preserve" text-anchor="start" x="281.1" y="-524.18" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">tier 1 · local disk</text>+<text xml:space="preserve" text-anchor="start" x="267.23" y="-512.08" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#8a93a6">is the .tw file on this box?</text>+</g>+<!-- req->local -->+<g id="edge1" class="edge">+<title>req->local</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M333.6,-582.07C333.6,-575.22 333.6,-567.68 333.6,-560.37"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="336.58,-560.56 333.6,-552.06 330.63,-560.56 336.58,-560.56"/>+</g>+<!-- obj -->+<g id="node3" class="node">+<title>obj</title>+<path fill="#151a24" stroke="#283041" stroke-width="1.3" d="M342.08,-453.52C342.08,-453.52 187.13,-453.52 187.13,-453.52 181.13,-453.52 175.13,-447.52 175.13,-441.52 175.13,-441.52 175.13,-413.6 175.13,-413.6 175.13,-407.6 181.13,-401.6 187.13,-401.6 187.13,-401.6 342.08,-401.6 342.08,-401.6 348.08,-401.6 354.08,-407.6 354.08,-413.6 354.08,-413.6 354.08,-441.52 354.08,-441.52 354.08,-447.52 348.08,-453.52 342.08,-453.52"/>+<text xml:space="preserve" text-anchor="start" x="196.73" y="-428.26" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">tier 2 · object storage</text>+<text xml:space="preserve" text-anchor="start" x="203.48" y="-416.16" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#8a93a6">is a prebuilt .tw stored?</text>+</g>+<!-- local->obj -->+<g id="edge3" class="edge">+<title>local->obj</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M315.11,-497.31C307.28,-486.66 298.05,-474.09 289.63,-462.62"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="292.07,-460.92 284.64,-455.83 287.27,-464.44 292.07,-460.92"/>+<text xml:space="preserve" text-anchor="middle" x="317.11" y="-471.07" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#8a93a6">  miss</text>+</g>+<!-- mmap -->+<g id="node7" class="node">+<title>mmap</title>+<path fill="#0c1714" stroke="#22e0a8" stroke-width="1.3" d="M378.58,-136.84C378.58,-136.84 166.62,-136.84 166.62,-136.84 160.62,-136.84 154.62,-130.84 154.62,-124.84 154.62,-124.84 154.62,-96.92 154.62,-96.92 154.62,-90.92 160.62,-84.92 166.62,-84.92 166.62,-84.92 378.58,-84.92 378.58,-84.92 384.58,-84.92 390.58,-90.92 390.58,-96.92 390.58,-96.92 390.58,-124.84 390.58,-124.84 390.58,-130.84 384.58,-136.84 378.58,-136.84"/>+<text xml:space="preserve" text-anchor="start" x="176.23" y="-111.58" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">mmap the .tw, search in place</text>+<text xml:space="preserve" text-anchor="start" x="176.6" y="-99.48" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#7ff0c8">fast: served from the OS page cache</text>+</g>+<!-- local->mmap -->+<g id="edge2" class="edge">+<title>local->mmap</title>+<path fill="none" stroke="#22e0a8" stroke-width="1.4" d="M362.98,-497.22C379.8,-479.86 397.6,-455.2 397.6,-428.56 397.6,-428.56 397.6,-428.56 397.6,-194.8 397.6,-171.99 382.97,-155.08 363.72,-142.71"/>+<polygon fill="#22e0a8" stroke="#22e0a8" stroke-width="1.4" points="365.24,-140.16 356.41,-138.41 362.23,-145.28 365.24,-140.16"/>+<text xml:space="preserve" text-anchor="middle" x="430.98" y="-327.19" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#7ff0c8">  hit (hot path)</text>+</g>+<!-- rebuild -->+<g id="node4" class="node">+<title>rebuild</title>+<path fill="#1b1115" stroke="#ff5a5f" stroke-width="1.3" d="M215.32,-357.6C215.32,-357.6 19.88,-357.6 19.88,-357.6 13.87,-357.6 7.88,-351.6 7.88,-345.6 7.88,-345.6 7.88,-317.68 7.88,-317.68 7.88,-311.68 13.88,-305.68 19.88,-305.68 19.88,-305.68 215.32,-305.68 215.32,-305.68 221.32,-305.68 227.32,-311.68 227.32,-317.68 227.32,-317.68 227.32,-345.6 227.32,-345.6 227.32,-351.6 221.32,-357.6 215.32,-357.6"/>+<text xml:space="preserve" text-anchor="start" x="56.1" y="-332.34" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">tier 3 · build it once</text>+<text xml:space="preserve" text-anchor="start" x="29.47" y="-320.24" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#ff9a9d">first crash of a brand-new release</text>+</g>+<!-- obj->rebuild -->+<g id="edge6" class="edge">+<title>obj->rebuild</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M178.37,-400.98C169.58,-396.58 161.12,-391.48 153.6,-385.6 147.01,-380.45 141.19,-373.75 136.24,-366.86"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="138.81,-365.34 131.64,-359.9 133.85,-368.63 138.81,-365.34"/>+<text xml:space="preserve" text-anchor="middle" x="194.1" y="-375.15" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#8a93a6">  miss (cold-cold)</text>+</g>+<!-- fetch -->+<g id="node5" class="node">+<title>fetch</title>+<path fill="#08161c" stroke="#00d4ff" stroke-width="1.3" d="M367.83,-289.68C367.83,-289.68 151.38,-289.68 151.38,-289.68 145.38,-289.68 139.38,-283.68 139.38,-277.68 139.38,-277.68 139.38,-249.76 139.38,-249.76 139.38,-243.76 145.38,-237.76 151.38,-237.76 151.38,-237.76 367.83,-237.76 367.83,-237.76 373.83,-237.76 379.83,-243.76 379.83,-249.76 379.83,-249.76 379.83,-277.68 379.83,-277.68 379.83,-283.68 373.83,-289.68 367.83,-289.68"/>+<text xml:space="preserve" text-anchor="start" x="160.98" y="-264.42" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">download the .tw, write to disk</text>+<text xml:space="preserve" text-anchor="start" x="220.98" y="-252.32" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#8fe6ff">still no parsing</text>+</g>+<!-- obj->fetch -->+<g id="edge4" class="edge">+<title>obj->fetch</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M263.82,-401.27C262.98,-374.15 261.64,-330.88 260.7,-300.33"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="263.69,-300.62 260.45,-292.21 257.74,-300.8 263.69,-300.62"/>+<text xml:space="preserve" text-anchor="middle" x="271.86" y="-375.15" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#8a93a6">  hit</text>+</g>+<!-- back -->+<g id="node6" class="node">+<title>back</title>+<path fill="#08161c" stroke="#00d4ff" stroke-width="1.3" d="M223.2,-221.76C223.2,-221.76 12,-221.76 12,-221.76 6,-221.76 0,-215.76 0,-209.76 0,-209.76 0,-181.84 0,-181.84 0,-175.84 6,-169.84 12,-169.84 12,-169.84 223.2,-169.84 223.2,-169.84 229.2,-169.84 235.2,-175.84 235.2,-181.84 235.2,-181.84 235.2,-209.76 235.2,-209.76 235.2,-215.76 229.2,-221.76 223.2,-221.76"/>+<text xml:space="preserve" text-anchor="start" x="21.6" y="-196.5" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">write the .tw to object storage</text>+<text xml:space="preserve" text-anchor="start" x="37.35" y="-184.4" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#8fe6ff">every replica is warm after this</text>+</g>+<!-- rebuild->back -->+<g id="edge7" class="edge">+<title>rebuild->back</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M117.6,-305.15C117.6,-284.59 117.6,-255.43 117.6,-232.54"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="120.58,-232.81 117.6,-224.31 114.63,-232.81 120.58,-232.81"/>+</g>+<!-- fetch->mmap -->+<g id="edge5" class="edge">+<title>fetch->mmap</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M261.81,-237.13C263.92,-212.57 267.14,-175.25 269.51,-147.73"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="272.46,-148.18 270.22,-139.45 266.53,-147.67 272.46,-148.18"/>+</g>+<!-- back->mmap -->+<g id="edge8" class="edge">+<title>back->mmap</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M165.37,-169.24C181.36,-160.69 199.33,-151.08 215.93,-142.2"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="217.03,-144.98 223.12,-138.35 214.22,-139.74 217.03,-144.98"/>+</g>+<!-- out -->+<g id="node8" class="node">+<title>out</title>+<path fill="#15121f" stroke="#7c5cff" stroke-width="1.3" d="M405.95,-51.92C405.95,-51.92 139.25,-51.92 139.25,-51.92 133.25,-51.92 127.25,-45.92 127.25,-39.92 127.25,-39.92 127.25,-12 127.25,-12 127.25,-6 133.25,0 139.25,0 139.25,0 405.95,0 405.95,0 411.95,0 417.95,-6 417.95,-12 417.95,-12 417.95,-39.92 417.95,-39.92 417.95,-45.92 411.95,-51.92 405.95,-51.92"/>+<text xml:space="preserve" text-anchor="start" x="224.23" y="-26.66" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#f4f6fb">resolved frame</text>+<text xml:space="preserve" text-anchor="start" x="148.85" y="-14.56" font-family="Menlo" font-size="12.00" fill="#b9a7ff">checkout.js:48 · validateCheckout</text>+</g>+<!-- mmap->out -->+<g id="edge9" class="edge">+<title>mmap->out</title>+<path fill="none" stroke="#5a6374" stroke-width="1.4" d="M272.6,-84.55C272.6,-77.7 272.6,-70.16 272.6,-62.85"/>+<polygon fill="#5a6374" stroke="#5a6374" stroke-width="1.4" points="275.58,-63.04 272.6,-54.54 269.63,-63.04 275.58,-63.04"/>+</g>+</g>+</svg>diff --git a/website/public/blog/system-design-of-a-symbolicator/tw-format.svg b/website/public/blog/system-design-of-a-symbolicator/tw-format.svgnew file mode 100644index 00000000..5ffd53b8--- /dev/null+++ b/website/public/blog/system-design-of-a-symbolicator/tw-format.svg@@ -0,0 +1,57 @@+<?xml version="1.0" encoding="UTF-8" standalone="no"?>+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">+<!-- Generated by graphviz version 15.0.0 (20260523.1842)+ -->+<!-- Title: tw_format Pages: 1 -->+<svg width="528pt" height="212pt"+ viewBox="0.00 0.00 528.00 212.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(10.8 201.55)">+<title>tw_format</title>+<!-- tw -->+<g id="node1" class="node">+<title>tw</title>+<polygon fill="#0d111a" stroke="none" points="8,-4 8,-186.75 498.25,-186.75 498.25,-4 8,-4"/>+<text xml:space="preserve" text-anchor="start" x="18" y="-163.5" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="15.00" fill="#f4f6fb">the .tw resolver format</text>+<text xml:space="preserve" text-anchor="start" x="181.5" y="-163.5" font-family="Helvetica,sans-Serif" font-size="14.00"> </text>+<text xml:space="preserve" text-anchor="start" x="189" y="-163.5" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#5a6374">flat · little-endian · memory-mapped</text>+<polygon fill="#11161f" stroke="none" points="9,-117.5 9,-152 95.25,-152 95.25,-117.5 9,-117.5"/>+<text xml:space="preserve" text-anchor="start" x="18" y="-129.95" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#c9d0dd">header</text>+<polygon fill="#11161f" stroke="none" points="95.25,-117.5 95.25,-152 435.75,-152 435.75,-117.5 95.25,-117.5"/>+<text xml:space="preserve" text-anchor="start" x="104.25" y="-130.25" font-family="Menlo" font-size="13.00" fill="#8a93a6">magic "TWSM" · version · table counts</text>+<polygon fill="#11161f" stroke="none" points="435.75,-117.5 435.75,-152 497.25,-152 497.25,-117.5 435.75,-117.5"/>+<text xml:space="preserve" text-anchor="start" x="463.5" y="-131.38" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#5a6374">24 B</text>+<polygon fill="#0e1322" stroke="none" points="9,-74 9,-117.5 95.25,-117.5 95.25,-74 9,-74"/>+<text xml:space="preserve" text-anchor="start" x="18" y="-90.95" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#b9a7ff">token[ ]</text>+<polygon fill="#0e1322" stroke="none" points="95.25,-74 95.25,-117.5 435.75,-117.5 435.75,-74 95.25,-74"/>+<text xml:space="preserve" text-anchor="start" x="104.25" y="-96.4" font-family="Menlo" font-size="13.00" fill="#c9d0dd">genLine genCol srcLine srcCol fileIdx fnIdx</text>+<text xml:space="preserve" text-anchor="start" x="157.12" y="-85.3" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#5a6374">sorted by (genLine, genCol) for binary search</text>+<polygon fill="#0e1322" stroke="none" points="435.75,-74 435.75,-117.5 497.25,-117.5 497.25,-74 435.75,-74"/>+<text xml:space="preserve" text-anchor="start" x="462.75" y="-98.75" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#8a93a6">24 B</text>+<text xml:space="preserve" text-anchor="start" x="462" y="-86" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#8a93a6">each</text>+<polygon fill="#0a1620" stroke="none" points="9,-39.5 9,-74 95.25,-74 95.25,-39.5 9,-39.5"/>+<text xml:space="preserve" text-anchor="start" x="18" y="-51.95" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#8fe6ff">file strings</text>+<polygon fill="#0a1620" stroke="none" points="95.25,-39.5 95.25,-74 435.75,-74 435.75,-39.5 95.25,-39.5"/>+<text xml:space="preserve" text-anchor="start" x="104.25" y="-52.25" font-family="Menlo" font-size="13.00" fill="#c9d0dd">offsets[ ] + blob</text>+<polygon fill="#0a1620" stroke="none" points="435.75,-39.5 435.75,-74 497.25,-74 497.25,-39.5 435.75,-39.5"/>+<text xml:space="preserve" text-anchor="start" x="444.75" y="-53.38" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#5a6374">interned</text>+<polygon fill="#0a1620" stroke="none" points="9,-5 9,-39.5 95.25,-39.5 95.25,-5 9,-5"/>+<text xml:space="preserve" text-anchor="start" x="18" y="-17.45" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="#8fe6ff">fn strings</text>+<polygon fill="#0a1620" stroke="none" points="95.25,-5 95.25,-39.5 435.75,-39.5 435.75,-5 95.25,-5"/>+<text xml:space="preserve" text-anchor="start" x="104.25" y="-17.75" font-family="Menlo" font-size="13.00" fill="#c9d0dd">offsets[ ] + blob</text>+<polygon fill="#0a1620" stroke="none" points="435.75,-5 435.75,-39.5 497.25,-39.5 497.25,-5 435.75,-5"/>+<text xml:space="preserve" text-anchor="start" x="444.75" y="-18.88" font-family="Helvetica,sans-Serif" font-size="12.00" fill="#5a6374">interned</text>+<polygon fill="#283041" stroke="#283041" points="8,-152 8,-152 498.25,-152 498.25,-152 8,-152"/>+<polygon fill="#283041" stroke="#283041" points="8,-117.5 8,-117.5 95.25,-117.5 95.25,-117.5 8,-117.5"/>+<polygon fill="#283041" stroke="#283041" points="95.25,-117.5 95.25,-117.5 435.75,-117.5 435.75,-117.5 95.25,-117.5"/>+<polygon fill="#283041" stroke="#283041" points="435.75,-117.5 435.75,-117.5 498.25,-117.5 498.25,-117.5 435.75,-117.5"/>+<polygon fill="#283041" stroke="#283041" points="8,-74 8,-74 95.25,-74 95.25,-74 8,-74"/>+<polygon fill="#283041" stroke="#283041" points="95.25,-74 95.25,-74 435.75,-74 435.75,-74 95.25,-74"/>+<polygon fill="#283041" stroke="#283041" points="435.75,-74 435.75,-74 498.25,-74 498.25,-74 435.75,-74"/>+<polygon fill="#283041" stroke="#283041" points="8,-39.5 8,-39.5 95.25,-39.5 95.25,-39.5 8,-39.5"/>+<polygon fill="#283041" stroke="#283041" points="95.25,-39.5 95.25,-39.5 435.75,-39.5 435.75,-39.5 95.25,-39.5"/>+<polygon fill="#283041" stroke="#283041" points="435.75,-39.5 435.75,-39.5 498.25,-39.5 498.25,-39.5 435.75,-39.5"/>+<polygon fill="none" stroke="#283041" points="8,-4 8,-186.75 498.25,-186.75 498.25,-4 8,-4"/>+</g>+</g>+</svg>