SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
admin: pre-gzip embedded static assets, add cache headers (#9918)
Chris Lue56a1c4c05be
diff --git a/weed/admin/Makefile b/weed/admin/Makefileindex 197644ed46..0cd9389145 100644--- a/weed/admin/Makefile+++ b/weed/admin/Makefile@@ -21,11 +21,17 @@ install-templ:# Generate templ files.PHONY: generate-generate: install-templ+generate: install-templ static-gz@echo "Generating templ files..."@templ generate ./view@echo "Generated: $(TEMPL_GO_FILES)"+# Regenerate the gzipped static mirror embedded into the binary+.PHONY: static-gz+static-gz:+ @echo "Generating gzipped static assets..."+ @go run gen_static_gz.go+# Clean generated files.PHONY: clean-templclean-templ:diff --git a/weed/admin/gen_static_gz.go b/weed/admin/gen_static_gz.gonew file mode 100644index 0000000000..d2542ae38b--- /dev/null+++ b/weed/admin/gen_static_gz.go@@ -0,0 +1,58 @@+//go:build ignore++// Regenerates static_gz/, the gzipped mirror of static/ that is embedded+// into the binary. Run after changing anything under static/:+//+// go generate ./weed/admin+package main++import (+ "bytes"+ "compress/gzip"+ "fmt"+ "io/fs"+ "os"+ "path/filepath"+)++func main() {+ if err := run(); err != nil {+ fmt.Fprintln(os.Stderr, err)+ os.Exit(1)+ }+}++func run() error {+ if err := os.RemoveAll("static_gz"); err != nil {+ return err+ }+ return filepath.WalkDir("static", func(p string, d fs.DirEntry, err error) error {+ if err != nil || d.IsDir() {+ return err+ }+ data, err := os.ReadFile(p)+ if err != nil {+ return err+ }+ var buf bytes.Buffer+ zw, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)+ if err != nil {+ return err+ }+ if _, err := zw.Write(data); err != nil {+ return err+ }+ if err := zw.Close(); err != nil {+ return err+ }+ rel, err := filepath.Rel("static", p)+ if err != nil {+ return err+ }+ out := filepath.Join("static_gz", rel+".gz")+ if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil {+ return err+ }+ return os.WriteFile(out, buf.Bytes(), 0644)+ })+}diff --git a/weed/admin/static_embed.go b/weed/admin/static_embed.goindex 1910afd86a..3a8dd5dd97 100644--- a/weed/admin/static_embed.go+++ b/weed/admin/static_embed.go@@ -2,13 +2,13 @@ package adminimport ("embed"- "io/fs")-//go:embed static/*-var StaticFS embed.FS+//go:generate go run gen_static_gz.go-// GetStaticFS returns the embedded static filesystem-func GetStaticFS() (fs.FS, error) {- return fs.Sub(StaticFS, "static")-}+// staticGzFS is the gzipped mirror of static/, generated by gen_static_gz.go.+// Only the compressed copies are embedded; StaticHandler decompresses for+// clients that do not accept gzip.+//+//go:embed all:static_gz+var staticGzFS embed.FSdiff --git a/weed/admin/static_gz/css/admin.css.gz b/weed/admin/static_gz/css/admin.css.gznew file mode 100644index 0000000000..70f6739e1cBinary files /dev/null and b/weed/admin/static_gz/css/admin.css.gz differdiff --git a/weed/admin/static_gz/css/bootstrap.min.css.gz b/weed/admin/static_gz/css/bootstrap.min.css.gznew file mode 100644index 0000000000..85170bd568Binary files /dev/null and b/weed/admin/static_gz/css/bootstrap.min.css.gz differdiff --git a/weed/admin/static_gz/css/bootstrap.min.css.map.gz b/weed/admin/static_gz/css/bootstrap.min.css.map.gznew file mode 100644index 0000000000..4216b68756Binary files /dev/null and b/weed/admin/static_gz/css/bootstrap.min.css.map.gz differdiff --git a/weed/admin/static_gz/css/fontawesome.min.css.gz b/weed/admin/static_gz/css/fontawesome.min.css.gznew file mode 100644index 0000000000..0983ddea7dBinary files /dev/null and b/weed/admin/static_gz/css/fontawesome.min.css.gz differdiff --git a/weed/admin/static_gz/favicon.ico.gz b/weed/admin/static_gz/favicon.ico.gznew file mode 100644index 0000000000..e2ca604e7dBinary files /dev/null and b/weed/admin/static_gz/favicon.ico.gz differdiff --git a/weed/admin/static_gz/js/admin.js.gz b/weed/admin/static_gz/js/admin.js.gznew file mode 100644index 0000000000..67efcbc799Binary files /dev/null and b/weed/admin/static_gz/js/admin.js.gz differdiff --git a/weed/admin/static_gz/js/bootstrap.bundle.min.js.gz b/weed/admin/static_gz/js/bootstrap.bundle.min.js.gznew file mode 100644index 0000000000..8dad8ca320Binary files /dev/null and b/weed/admin/static_gz/js/bootstrap.bundle.min.js.gz differdiff --git a/weed/admin/static_gz/js/bootstrap.bundle.min.js.map.gz b/weed/admin/static_gz/js/bootstrap.bundle.min.js.map.gznew file mode 100644index 0000000000..cc914cfa62Binary files /dev/null and b/weed/admin/static_gz/js/bootstrap.bundle.min.js.map.gz differdiff --git a/weed/admin/static_gz/js/htmx.min.js.gz b/weed/admin/static_gz/js/htmx.min.js.gznew file mode 100644index 0000000000..3e8bd843d0Binary files /dev/null and b/weed/admin/static_gz/js/htmx.min.js.gz differdiff --git a/weed/admin/static_gz/js/iam-utils.js.gz b/weed/admin/static_gz/js/iam-utils.js.gznew file mode 100644index 0000000000..94146f8a1cBinary files /dev/null and b/weed/admin/static_gz/js/iam-utils.js.gz differdiff --git a/weed/admin/static_gz/js/modal-alerts.js.gz b/weed/admin/static_gz/js/modal-alerts.js.gznew file mode 100644index 0000000000..3dfccb61d8Binary files /dev/null and b/weed/admin/static_gz/js/modal-alerts.js.gz differdiff --git a/weed/admin/static_gz/js/s3tables.js.gz b/weed/admin/static_gz/js/s3tables.js.gznew file mode 100644index 0000000000..f8430e4f81Binary files /dev/null and b/weed/admin/static_gz/js/s3tables.js.gz differdiff --git a/weed/admin/static_gz/webfonts/fa-brands-400.ttf.gz b/weed/admin/static_gz/webfonts/fa-brands-400.ttf.gznew file mode 100644index 0000000000..671c00b798Binary files /dev/null and b/weed/admin/static_gz/webfonts/fa-brands-400.ttf.gz differdiff --git a/weed/admin/static_gz/webfonts/fa-brands-400.woff2.gz b/weed/admin/static_gz/webfonts/fa-brands-400.woff2.gznew file mode 100644index 0000000000..49b4342ffeBinary files /dev/null and b/weed/admin/static_gz/webfonts/fa-brands-400.woff2.gz differdiff --git a/weed/admin/static_gz/webfonts/fa-regular-400.ttf.gz b/weed/admin/static_gz/webfonts/fa-regular-400.ttf.gznew file mode 100644index 0000000000..d1524c81caBinary files /dev/null and b/weed/admin/static_gz/webfonts/fa-regular-400.ttf.gz differdiff --git a/weed/admin/static_gz/webfonts/fa-regular-400.woff2.gz b/weed/admin/static_gz/webfonts/fa-regular-400.woff2.gznew file mode 100644index 0000000000..60a602df80Binary files /dev/null and b/weed/admin/static_gz/webfonts/fa-regular-400.woff2.gz differdiff --git a/weed/admin/static_gz/webfonts/fa-solid-900.ttf.gz b/weed/admin/static_gz/webfonts/fa-solid-900.ttf.gznew file mode 100644index 0000000000..009edbc6dcBinary files /dev/null and b/weed/admin/static_gz/webfonts/fa-solid-900.ttf.gz differdiff --git a/weed/admin/static_gz/webfonts/fa-solid-900.woff2.gz b/weed/admin/static_gz/webfonts/fa-solid-900.woff2.gznew file mode 100644index 0000000000..98a96cc4a0Binary files /dev/null and b/weed/admin/static_gz/webfonts/fa-solid-900.woff2.gz differdiff --git a/weed/admin/static_handler.go b/weed/admin/static_handler.gonew file mode 100644index 0000000000..34dd15f371--- /dev/null+++ b/weed/admin/static_handler.go@@ -0,0 +1,116 @@+package admin++import (+ "bytes"+ "compress/gzip"+ "crypto/sha256"+ "encoding/hex"+ "io"+ "io/fs"+ "net/http"+ "path"+ "strconv"+ "strings"+ "sync"+ "time"+)++// Assets only change with the binary; the ETag revalidates across upgrades.+const staticCacheControl = "public, max-age=3600"++type staticAsset struct {+ name string // base name, drives Content-Type in http.ServeContent+ gz []byte+ etag string+ etagGz string+}++// identity returns the uncompressed bytes.+func (a *staticAsset) identity() ([]byte, error) {+ zr, err := gzip.NewReader(bytes.NewReader(a.gz))+ if err != nil {+ return nil, err+ }+ defer zr.Close()+ return io.ReadAll(zr)+}++var staticAssets = sync.OnceValue(func() map[string]*staticAsset {+ assets := make(map[string]*staticAsset)+ err := fs.WalkDir(staticGzFS, "static_gz", func(p string, d fs.DirEntry, err error) error {+ if err != nil || d.IsDir() {+ return err+ }+ gz, err := fs.ReadFile(staticGzFS, p)+ if err != nil {+ return err+ }+ key := strings.TrimSuffix(strings.TrimPrefix(p, "static_gz/"), ".gz")+ sum := sha256.Sum256(gz)+ hash := hex.EncodeToString(sum[:8])+ assets[key] = &staticAsset{+ name: path.Base(key),+ gz: gz,+ etag: `"` + hash + `"`,+ etagGz: `"` + hash + `-gz"`,+ }+ return nil+ })+ if err != nil {+ panic("walk embedded static assets: " + err.Error())+ }+ return assets+})++// StaticHandler serves the embedded admin static assets, which are gzipped+// at generation time. Gzip-capable clients get the compressed bytes as-is;+// others get them decompressed on the fly.+func StaticHandler() http.Handler {+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ key := strings.TrimPrefix(path.Clean("/"+r.URL.Path), "/")+ asset, ok := staticAssets()[key]+ if !ok {+ http.NotFound(w, r)+ return+ }+ h := w.Header()+ h.Set("Cache-Control", staticCacheControl)+ h.Add("Vary", "Accept-Encoding")+ if acceptsGzip(r) {+ h.Set("Content-Encoding", "gzip")+ h.Set("ETag", asset.etagGz)+ // ServeContent skips Content-Length when Content-Encoding is set+ if r.Header.Get("Range") == "" {+ h.Set("Content-Length", strconv.Itoa(len(asset.gz)))+ }+ http.ServeContent(w, r, asset.name, time.Time{}, bytes.NewReader(asset.gz))+ return+ }+ data, err := asset.identity()+ if err != nil {+ http.Error(w, err.Error(), http.StatusInternalServerError)+ return+ }+ h.Set("ETag", asset.etag)+ http.ServeContent(w, r, asset.name, time.Time{}, bytes.NewReader(data))+ })+}++func acceptsGzip(r *http.Request) bool {+ for _, part := range strings.Split(r.Header.Get("Accept-Encoding"), ",") {+ token, attr, hasAttr := strings.Cut(strings.TrimSpace(part), ";")+ if strings.TrimSpace(token) != "gzip" {+ continue+ }+ if !hasAttr {+ return true+ }+ q, ok := strings.CutPrefix(strings.TrimSpace(attr), "q=")+ if !ok {+ return true+ }+ v, err := strconv.ParseFloat(q, 64)+ return err == nil && v > 0+ }+ return false+}diff --git a/weed/admin/static_handler_test.go b/weed/admin/static_handler_test.gonew file mode 100644index 0000000000..1875cb3ece--- /dev/null+++ b/weed/admin/static_handler_test.go@@ -0,0 +1,215 @@+package admin++import (+ "bytes"+ "compress/gzip"+ "io"+ "io/fs"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++// TestStaticGzMirror fails when static_gz/ is out of sync with static/.+// Regenerate with: go generate ./weed/admin+func TestStaticGzMirror(t *testing.T) {+ sources := make(map[string][]byte)+ err := filepath.WalkDir("static", func(p string, d fs.DirEntry, err error) error {+ if err != nil || d.IsDir() {+ return err+ }+ data, err := os.ReadFile(p)+ if err != nil {+ return err+ }+ rel, err := filepath.Rel("static", p)+ if err != nil {+ return err+ }+ sources[filepath.ToSlash(rel)] = data+ return nil+ })+ if err != nil {+ t.Fatal(err)+ }+ if len(sources) == 0 {+ t.Fatal("no files under static/")+ }++ mirrored := 0+ err = filepath.WalkDir("static_gz", func(p string, d fs.DirEntry, err error) error {+ if err != nil || d.IsDir() {+ return err+ }+ rel, err := filepath.Rel("static_gz", p)+ if err != nil {+ return err+ }+ key := strings.TrimSuffix(filepath.ToSlash(rel), ".gz")+ want, ok := sources[key]+ if !ok {+ t.Errorf("static_gz/%s has no source in static/", rel)+ return nil+ }+ mirrored+++ gz, err := os.ReadFile(p)+ if err != nil {+ return err+ }+ zr, err := gzip.NewReader(bytes.NewReader(gz))+ if err != nil {+ return err+ }+ defer zr.Close()+ got, err := io.ReadAll(zr)+ if err != nil {+ return err+ }+ if !bytes.Equal(got, want) {+ t.Errorf("static_gz/%s does not match static/%s", rel, key)+ }+ return nil+ })+ if err != nil {+ t.Fatal(err)+ }+ if mirrored != len(sources) {+ t.Errorf("static_gz mirrors %d of %d files under static/", mirrored, len(sources))+ }+}++func fetchStatic(t *testing.T, path string, header map[string]string) *http.Response {+ t.Helper()+ req := httptest.NewRequest("GET", path, nil)+ for k, v := range header {+ req.Header.Set(k, v)+ }+ w := httptest.NewRecorder()+ StaticHandler().ServeHTTP(w, req)+ return w.Result()+}++func TestStaticHandlerGzip(t *testing.T) {+ resp := fetchStatic(t, "/css/bootstrap.min.css", map[string]string{"Accept-Encoding": "gzip, deflate, br"})+ if resp.StatusCode != http.StatusOK {+ t.Fatalf("status %d", resp.StatusCode)+ }+ if got := resp.Header.Get("Content-Encoding"); got != "gzip" {+ t.Fatalf("Content-Encoding %q", got)+ }+ if resp.Header.Get("Cache-Control") != staticCacheControl {+ t.Errorf("Cache-Control %q", resp.Header.Get("Cache-Control"))+ }+ if resp.Header.Get("Vary") != "Accept-Encoding" {+ t.Errorf("Vary %q", resp.Header.Get("Vary"))+ }+ if resp.Header.Get("ETag") == "" {+ t.Error("missing ETag")+ }+ if got := resp.Header.Get("Content-Type"); !strings.HasPrefix(got, "text/css") {+ t.Errorf("Content-Type %q", got)+ }+ body, err := io.ReadAll(resp.Body)+ if err != nil {+ t.Fatal(err)+ }+ if resp.Header.Get("Content-Length") == "" {+ t.Error("missing Content-Length")+ }+ zr, err := gzip.NewReader(bytes.NewReader(body))+ if err != nil {+ t.Fatal(err)+ }+ defer zr.Close()+ got, err := io.ReadAll(zr)+ if err != nil {+ t.Fatal(err)+ }+ want, err := os.ReadFile("static/css/bootstrap.min.css")+ if err != nil {+ t.Fatal(err)+ }+ if !bytes.Equal(got, want) {+ t.Error("gzip body does not decompress to the source file")+ }+}++func TestStaticHandlerIdentity(t *testing.T) {+ resp := fetchStatic(t, "/css/bootstrap.min.css", nil)+ if resp.StatusCode != http.StatusOK {+ t.Fatalf("status %d", resp.StatusCode)+ }+ if got := resp.Header.Get("Content-Encoding"); got != "" {+ t.Fatalf("Content-Encoding %q", got)+ }+ body, err := io.ReadAll(resp.Body)+ if err != nil {+ t.Fatal(err)+ }+ want, err := os.ReadFile("static/css/bootstrap.min.css")+ if err != nil {+ t.Fatal(err)+ }+ if !bytes.Equal(body, want) {+ t.Error("identity body does not match the source file")+ }+}++func TestStaticHandlerNotModified(t *testing.T) {+ first := fetchStatic(t, "/css/admin.css", map[string]string{"Accept-Encoding": "gzip"})+ etag := first.Header.Get("ETag")+ if etag == "" {+ t.Fatal("missing ETag")+ }+ resp := fetchStatic(t, "/css/admin.css", map[string]string{"Accept-Encoding": "gzip", "If-None-Match": etag})+ if resp.StatusCode != http.StatusNotModified {+ t.Fatalf("status %d", resp.StatusCode)+ }+ if resp.Header.Get("Content-Length") != "" {+ t.Errorf("304 carries Content-Length %q", resp.Header.Get("Content-Length"))+ }+}++func TestStaticHandlerVariantETags(t *testing.T) {+ gz := fetchStatic(t, "/css/admin.css", map[string]string{"Accept-Encoding": "gzip"})+ id := fetchStatic(t, "/css/admin.css", nil)+ if gz.Header.Get("ETag") == id.Header.Get("ETag") {+ t.Error("gzip and identity variants share an ETag")+ }+}++func TestStaticHandlerNotFound(t *testing.T) {+ for _, p := range []string{"/css/missing.css", "/css/", "/", "/../static_embed.go"} {+ if resp := fetchStatic(t, p, nil); resp.StatusCode != http.StatusNotFound {+ t.Errorf("%s: status %d", p, resp.StatusCode)+ }+ }+}++func TestAcceptsGzip(t *testing.T) {+ cases := []struct {+ header string+ want bool+ }{+ {"", false},+ {"gzip", true},+ {"gzip, deflate, br", true},+ {"br;q=1.0, gzip;q=0.8", true},+ {"gzip;q=0", false},+ {"gzip;q=0.0", false},+ {"deflate", false},+ {"x-gzip-like", false},+ }+ for _, c := range cases {+ r := httptest.NewRequest("GET", "/", nil)+ if c.header != "" {+ r.Header.Set("Accept-Encoding", c.header)+ }+ if got := acceptsGzip(r); got != c.want {+ t.Errorf("acceptsGzip(%q) = %v, want %v", c.header, got, c.want)+ }+ }+}diff --git a/weed/command/admin.go b/weed/command/admin.goindex f823507bc3..c3a699082a 100644--- a/weed/command/admin.go+++ b/weed/command/admin.go@@ -360,15 +360,9 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,SameSite: http.SameSiteLaxMode,}- // Static files - serve from embedded filesystem- staticFS, err := admin.GetStaticFS()- if err != nil {- log.Printf("Warning: Failed to load embedded static files: %v", err)- } else {- staticHandler := http.FileServer(http.FS(staticFS))- r.Handle("/static", http.RedirectHandler("/static/", http.StatusMovedPermanently))- r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", staticHandler))- }+ // Static files - pre-gzipped and embedded in the binary+ r.Handle("/static", http.RedirectHandler("/static/", http.StatusMovedPermanently))+ r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", admin.StaticHandler()))// Create admin server (plugin is always enabled)adminServer := dash.NewAdminServer(*options.master, nil, dataDir, icebergPort)