The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
package main
import (
"bytes"
"debug/macho"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
)
type corpus struct {
Language string `json:"language"`
Urls []string `json:"urls"`
}
type dartBuild struct {
BuildID string `json:"buildId"`
Trace string `json:"trace"`
}
type dartCorpus struct {
Language string `json:"language"`
BinaryName string `json:"binaryName,omitempty"`
Builds []dartBuild `json:"builds"`
}
const vlqChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
var vlqIndex = func() [128]int8 {
var t [128]int8
for i := range t {
t[i] = -1
}
for i, c := range vlqChars {
t[c] = int8(i)
}
return t
}()
func vlqEncode(b *strings.Builder, v int) {
u := v << 1
if v < 0 {
u = (-v << 1) | 1
}
for {
digit := u & 31
u >>= 5
if u > 0 {
digit |= 32
}
b.WriteByte(vlqChars[digit])
if u == 0 {
break
}
}
}
func mappingsEndState(mappings string) (srcIdx, srcLine, srcCol, nameIdx int) {
vals := make([]int, 0, 5)
cur, shift := 0, 0
flush := func() {
if len(vals) >= 4 {
srcIdx += vals[1]
srcLine += vals[2]
srcCol += vals[3]
}
if len(vals) >= 5 {
nameIdx += vals[4]
}
vals = vals[:0]
}
for _, c := range mappings {
if c == ';' || c == ',' {
flush()
continue
}
d := vlqIndex[c]
cur |= int(d&31) << shift
if d&32 != 0 {
shift += 5
continue
}
v := cur >> 1
if cur&1 != 0 {
v = -v
}
vals = append(vals, v)
cur, shift = 0, 0
}
flush()
return
}
func padMap(mapBytes []byte, kb, mappingsKB, seed int) []byte {
var m map[string]any
if err := json.Unmarshal(mapBytes, &m); err != nil {
panic(err)
}
sources, _ := m["sources"].([]any)
content, _ := m["sourcesContent"].([]any)
for len(content) < len(sources) {
content = append(content, nil)
}
padIdx := len(sources)
var pad strings.Builder
line := fmt.Sprintf("const benchPadValue%d = %d;\n", seed, seed)
for pad.Len() < kb*1024 {
pad.WriteString(line)
}
m["sources"] = append(sources, "__benchpad.js")
m["sourcesContent"] = append(content, pad.String())
if mappingsKB > 0 {
mappings, _ := m["mappings"].(string)
names, _ := m["names"].([]any)
nameBase := len(names)
for i := 0; i < 64; i++ {
names = append(names, fmt.Sprintf("__benchFn%d_%d", seed, i))
}
m["names"] = names
srcIdx, srcLine, srcCol, nameIdx := mappingsEndState(mappings)
var b strings.Builder
b.WriteString(mappings)
dSrc := padIdx - srcIdx
dLine := -srcLine
dCol := -srcCol
dName := nameBase - nameIdx
genLine := 0
for b.Len() < len(mappings)+mappingsKB*1024 {
b.WriteByte(';')
genLine++
col := 0
for s := 0; s < 8; s++ {
if s > 0 {
b.WriteByte(',')
}
vlqEncode(&b, col)
vlqEncode(&b, dSrc)
vlqEncode(&b, dLine)
vlqEncode(&b, dCol)
vlqEncode(&b, dName)
dSrc, dCol = 0, 1
dLine = 0
dName = 1
if s == 7 {
dLine, dCol = 1, -7
dName = -7
}
col = 10
}
}
m["mappings"] = b.String()
}
out, err := json.Marshal(m)
if err != nil {
panic(err)
}
return out
}
func main() {
language := flag.String("language", "js", "corpus language: js or dart")
bundle := flag.String("bundle", "../../testing/symbolication/node-app/dist/app.mjs", "")
mapFile := flag.String("map", "../../testing/symbolication/node-app/dist/app.mjs.map", "")
symbols := flag.String("symbols", "seeds/dart/app.debug.elf", "dart: seed .symbols/.elf")
dsym := flag.String("dsym", "seeds/ios/app.dsym", "ios: seed dSYM (Mach-O DWARF)")
binary := flag.String("binary", "sample", "honeycomb-ios: Mach-O name inside the .dSYM bundle")
mapping := flag.String("mapping", "seeds/android/mapping.txt", "android: seed R8 mapping.txt")
traceFile := flag.String("trace", "seeds/dart/trace.txt", "dart/ios/android: seed non-symbolic trace")
entries := flag.Int("entries", 1, "")
padKB := flag.Int("pad-kb", 256, "")
mapPadKB := flag.Int("map-pad-kb", 0, "")
mappingsPadKB := flag.Int("mappings-pad-kb", 0, "")
out := flag.String("out", "./corpus", "")
flag.Parse()
if err := os.MkdirAll(*out, 0o755); err != nil {
panic(err)
}
if *language == "dart" {
generateDart(*out, *symbols, *traceFile, *entries)
return
}
if *language == "ios" {
generateIOS(*out, *dsym, *traceFile, *entries)
return
}
if *language == "honeycomb-ios" {
generateHoneycombIOS(*out, *dsym, *traceFile, *binary, *entries)
return
}
if *language == "android" {
generateAndroid(*out, *mapping, *traceFile, *mapPadKB, *entries)
return
}
bundleBytes, err := os.ReadFile(*bundle)
if err != nil {
panic(err)
}
mapBytes, err := os.ReadFile(*mapFile)
if err != nil {
panic(err)
}
firstLine := strings.SplitN(string(bundleBytes), "\n", 2)[0]
c := corpus{Language: "js"}
for n := 0; n < *entries; n++ {
var pad strings.Builder
chunk := "function __benchPad%d_%d(a,b){var c=a*b+%d;for(var i=0;i<3;i++){c+=i*a-b}return c}\n"
i := 0
for pad.Len() < *padKB*1024 {
pad.WriteString(fmt.Sprintf(chunk, n, i, i+n))
i++
}
name := fmt.Sprintf("app%d.mjs", n)
content := firstLine + "\n" + pad.String() + "//# sourceMappingURL=" + name + ".map\n"
if err := os.WriteFile(filepath.Join(*out, name), []byte(content), 0o644); err != nil {
panic(err)
}
entryMap := mapBytes
if *mapPadKB > 0 || *mappingsPadKB > 0 {
entryMap = padMap(mapBytes, *mapPadKB, *mappingsPadKB, n)
}
if err := os.WriteFile(filepath.Join(*out, name+".map"), entryMap, 0o644); err != nil {
panic(err)
}
c.Urls = append(c.Urls, name)
}
data, _ := json.MarshalIndent(c, "", " ")
if err := os.WriteFile(filepath.Join(*out, "corpus.json"), data, 0o644); err != nil {
panic(err)
}
fmt.Printf("wrote %d entries (%d KB padding each) to %s\n", *entries, *padKB, *out)
}
var archRe = regexp.MustCompile(`os:\s*\S+\s+arch:\s*(\S+)`)
var buildIDLineRe = regexp.MustCompile(`build_id:\s*'[0-9a-fA-F]+'`)
func generateDart(out, symbolsPath, tracePath string, entries int) {
elf, err := os.ReadFile(symbolsPath)
if err != nil {
panic(fmt.Errorf("reading dart seed symbols %q: %w", symbolsPath, err))
}
template, err := os.ReadFile(tracePath)
if err != nil {
panic(fmt.Errorf("reading dart seed trace %q: %w", tracePath, err))
}
absSeed, err := filepath.Abs(symbolsPath)
if err != nil {
panic(err)
}
arch := "arm64"
if m := archRe.FindStringSubmatch(string(template)); m != nil {
arch = m[1]
}
c := dartCorpus{Language: "dart"}
for n := 0; n < entries; n++ {
buildID := fmt.Sprintf("%032x", n)
dest := filepath.Join(out, buildID+"-"+arch+".symbols")
_ = os.Remove(dest)
if err := os.Link(absSeed, dest); err != nil {
if werr := os.WriteFile(dest, elf, 0o644); werr != nil {
panic(werr)
}
}
c.Builds = append(c.Builds, dartBuild{BuildID: buildID, Trace: substituteBuildID(string(template), buildID)})
}
data, _ := json.MarshalIndent(c, "", " ")
if err := os.WriteFile(filepath.Join(out, "corpus.json"), data, 0o644); err != nil {
panic(err)
}
fmt.Printf("wrote %d dart builds (arch %s, seed %s) to %s\n", entries, arch, symbolsPath, out)
}
func substituteBuildID(trace, buildID string) string {
if buildIDLineRe.MatchString(trace) {
return buildIDLineRe.ReplaceAllString(trace, "build_id: '"+buildID+"'")
}
return "build_id: '" + buildID + "'\n" + trace
}
var iosFrameUUIDRe = regexp.MustCompile(`(#\d+\s+)[0-9a-fA-F]{32}(\s+0x)`)
func generateIOS(out, dsymPath, tracePath string, entries int) {
dsym, err := os.ReadFile(dsymPath)
if err != nil {
panic(fmt.Errorf("reading ios seed dSYM %q: %w", dsymPath, err))
}
template, err := os.ReadFile(tracePath)
if err != nil {
panic(fmt.Errorf("reading ios seed trace %q: %w", tracePath, err))
}
absSeed, err := filepath.Abs(dsymPath)
if err != nil {
panic(err)
}
arch := "arm64"
if m := archRe.FindStringSubmatch(string(template)); m != nil {
arch = m[1]
}
c := dartCorpus{Language: "ios"}
for n := 0; n < entries; n++ {
uuid := fmt.Sprintf("%032x", n)
dest := filepath.Join(out, uuid+".dsym")
_ = os.Remove(dest)
if err := os.Link(absSeed, dest); err != nil {
if werr := os.WriteFile(dest, dsym, 0o644); werr != nil {
panic(werr)
}
}
c.Builds = append(c.Builds, dartBuild{BuildID: uuid, Trace: substituteIOSUUID(string(template), uuid)})
}
data, _ := json.MarshalIndent(c, "", " ")
if err := os.WriteFile(filepath.Join(out, "corpus.json"), data, 0o644); err != nil {
panic(err)
}
fmt.Printf("wrote %d ios builds (arch %s, seed %s) to %s\n", entries, arch, dsymPath, out)
}
func substituteIOSUUID(trace, uuid string) string {
return iosFrameUUIDRe.ReplaceAllString(trace, "${1}"+uuid+"${2}")
}
func generateAndroid(out, mappingPath, tracePath string, mapPadKB, entries int) {
base, err := os.ReadFile(mappingPath)
if err != nil {
panic(fmt.Errorf("reading android seed mapping %q: %w", mappingPath, err))
}
trace, err := os.ReadFile(tracePath)
if err != nil {
panic(fmt.Errorf("reading android seed trace %q: %w", tracePath, err))
}
absSeed, err := filepath.Abs(mappingPath)
if err != nil {
panic(err)
}
c := dartCorpus{Language: "android"}
for n := 0; n < entries; n++ {
uuid := fmt.Sprintf("%032x", n)
dest := filepath.Join(out, uuid+".txt")
_ = os.Remove(dest)
if mapPadKB > 0 {
if err := os.WriteFile(dest, padMapping(base, mapPadKB, n), 0o644); err != nil {
panic(err)
}
} else if err := os.Link(absSeed, dest); err != nil {
if werr := os.WriteFile(dest, base, 0o644); werr != nil {
panic(werr)
}
}
c.Builds = append(c.Builds, dartBuild{BuildID: uuid, Trace: string(trace)})
}
data, _ := json.MarshalIndent(c, "", " ")
if err := os.WriteFile(filepath.Join(out, "corpus.json"), data, 0o644); err != nil {
panic(err)
}
fmt.Printf("wrote %d android builds (map-pad %d KB, seed %s) to %s\n", entries, mapPadKB, mappingPath, out)
}
func padMapping(base []byte, kb, seed int) []byte {
var b strings.Builder
b.Write(base)
if len(base) > 0 && base[len(base)-1] != '\n' {
b.WriteByte('\n')
}
target := len(base) + kb*1024
for i := 0; b.Len() < target; i++ {
fmt.Fprintf(&b, "benchpad.S%dClass%d -> twbp_%d_%d:\n", seed, i, seed, i)
b.WriteString(" 1:1:void run():1:1 -> a\n")
b.WriteString(" 2:2:int compute(int):2:2 -> b\n")
}
return []byte(b.String())
}
var twFrameRe = regexp.MustCompile(`^\s*#(\d+)\s+([0-9a-fA-F]{32})\s+0x([0-9a-fA-F]+)\s+(\S+)\s*$`)
func hyphenateUUID(hex32 string) string {
h := strings.ToUpper(hex32)
if len(h) != 32 {
return h
}
return h[0:8] + "-" + h[8:12] + "-" + h[12:16] + "-" + h[16:20] + "-" + h[20:32]
}
func seedMachoUUID(data []byte) []byte {
f, err := macho.NewFile(bytes.NewReader(data))
if err != nil {
return nil
}
defer f.Close()
for _, l := range f.Loads {
lb, ok := l.(macho.LoadBytes)
if !ok {
continue
}
raw := lb.Raw()
if len(raw) >= 24 && f.ByteOrder.Uint32(raw[0:4]) == 0x1b {
return append([]byte(nil), raw[8:24]...)
}
}
return nil
}
func patchMachoUUID(seed, oldUUID, newUUID []byte) []byte {
out := make([]byte, len(seed))
copy(out, seed)
if i := bytes.Index(out, oldUUID); i >= 0 {
copy(out[i:i+16], newUUID)
}
return out
}
type iosSeedFrame struct {
idx int
uuid32 string
image string
offset uint64
}
func generateHoneycombIOS(out, dsymPath, tracePath, binaryName string, entries int) {
seed, err := os.ReadFile(dsymPath)
if err != nil {
panic(fmt.Errorf("reading ios seed dSYM %q: %w", dsymPath, err))
}
template, err := os.ReadFile(tracePath)
if err != nil {
panic(fmt.Errorf("reading ios seed trace %q: %w", tracePath, err))
}
oldUUID := seedMachoUUID(seed)
if oldUUID == nil {
panic(fmt.Errorf("seed dSYM %q has no LC_UUID", dsymPath))
}
arch := "arm64"
if m := archRe.FindStringSubmatch(string(template)); m != nil {
arch = m[1]
}
var frames []iosSeedFrame
for _, line := range strings.Split(string(template), "\n") {
m := twFrameRe.FindStringSubmatch(line)
if m == nil {
continue
}
idx, _ := strconv.Atoi(m[1])
off, _ := strconv.ParseUint(m[3], 16, 64)
frames = append(frames, iosSeedFrame{idx: idx, uuid32: strings.ToLower(m[2]), image: m[4], offset: off})
}
if len(frames) == 0 {
panic(fmt.Errorf("ios seed trace %q has no frames", tracePath))
}
c := dartCorpus{Language: "honeycomb-ios", BinaryName: binaryName}
for n := 0; n < entries; n++ {
buildHex := fmt.Sprintf("%032x", n)
buildUUID := hyphenateUUID(buildHex)
newUUID, _ := hex.DecodeString(buildHex)
dwarfDir := filepath.Join(out, buildUUID+".dSYM", "Contents", "Resources", "DWARF")
if err := os.MkdirAll(dwarfDir, 0o755); err != nil {
panic(err)
}
if err := os.WriteFile(filepath.Join(dwarfDir, binaryName), patchMachoUUID(seed, oldUUID, newUUID), 0o644); err != nil {
panic(err)
}
var b strings.Builder
fmt.Fprintf(&b, "os: ios arch: %s\n", arch)
for _, f := range frames {
uuid := buildUUID
if f.uuid32 != frames[0].uuid32 {
uuid = hyphenateUUID(f.uuid32)
}
fmt.Fprintf(&b, "%d %s 0x%016x %s + %d\n", f.idx, f.image, 0x100000000+f.offset, uuid, f.offset)
}
c.Builds = append(c.Builds, dartBuild{BuildID: buildUUID, Trace: b.String()})
}
data, _ := json.MarshalIndent(c, "", " ")
if err := os.WriteFile(filepath.Join(out, "corpus.json"), data, 0o644); err != nil {
panic(err)
}
fmt.Printf("wrote %d honeycomb-ios builds (arch %s, binary %s, seed %s) to %s\n", entries, arch, binaryName, dsymPath, out)
}