Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
davsync: a GitHub backup too large for the Contents API goes up as a blob (Zhang-Wei, Discord)
yetone93b87d00044d
diff --git a/docs/subsystems/sync-backup.md b/docs/subsystems/sync-backup.mdindex cbf52fc02..ab2531b06 100644--- a/docs/subsystems/sync-backup.md+++ b/docs/subsystems/sync-backup.md@@ -12,8 +12,8 @@ clone a working tree or synchronize arbitrary files.versions, notices and undo file in `sync-state.json`. These files live besidemagpie's settings. Secrets are local, never in the status response.- The [`remote`](../../internal/davsync/dav.go) interface reads a version and- conditionally replaces it. WebDAV and S3 use ETags; GitHub uses the blob SHA- required by its Contents API. [`github.go`](../../internal/davsync/github.go)+ conditionally replaces it. WebDAV and S3 use ETags; GitHub uses the file's+ blob SHA. [`github.go`](../../internal/davsync/github.go)interprets `github://owner/repo[/prefix]`, with `Config.Branch` selecting abranch or using the repository's default.- [`backup.go`](../../internal/backup/backup.go) owns bundle contents and@@ -97,12 +97,23 @@ repository can keep the token. The branch is part of the sync state key.repository read doesn't say `"private": true`, since a public history handsevery sealed version to anyone to guess the passphrase against(`TestGitHubPublicRepositoryRefused`). In an empty repository, the first encrypted- backup initializes its default branch through the Contents API. A selected+ backup initializes its default branch through the Contents API. When that+ API refuses it as too large, a short `README.md` in magpie's folder makes+ the first commit and the backup follows as the second. A selectedbranch other than the default must already exist. The token needs repositoryContents read and write.- Backups are `<prefix>/magpie/magpie.magpie-backup`; optional usage files- live beside them under `usage/`. Changes create commits, and unchanged- setup does not create another backup commit.+ live beside them under `usage/`. Each write or removal is one commit, and+ unchanged setup does not create another backup commit.+- A repository with commits is written through the Git Data API: a blob+ (up to 100 MB), a tree on the head commit's tree, a commit with the head+ as its parent, and a ref update without force. The Contents API's PUT+ refuses files well under that with 422 "the file is too large to be+ processed" (a 102-skill backup past ~19 MB, Discord). The file must still+ be at the version sync read, checked at the head commit, before the+ commit is built. A ref update that is not a fast-forward is built again on+ the new head, up to three times; if the new head changed the file it is+ `errChanged`. A branch deleted meanwhile is an error, never created.- Tokens are separate from the library's GitHub token, sent only to`https://api.github.com`, and must differ from the encryption passphrase.Repository history retains older encrypted versions.@@ -115,7 +126,8 @@ repository can keep the token. The branch is part of the sync state key.- Large files are downloaded by blob SHA, preserving the version that wasread. Sync files are limited to 64 MiB. Usage listing fails at the ContentsAPI's 1,000-file limit rather than treating a truncated list as complete.-- A stale SHA or a racing initial creation returns `errChanged`. Other 422+- A stale SHA, a backup another commit changed, or a racing initial+ creation returns `errChanged`. Other 422responses stay actionable errors. Authentication, permission and branchfailures remain distinct from missing files. Rate limits use sync backoff.- Request archival remains an S3-only feature; its objects are separate from@@ -132,7 +144,11 @@ node --test internal/gui/tests/github-sync.test.cjs internal/gui/tests/s3-sync.t`TestGitHubEmptyRepository` covers the first encrypted backup initializing anempty repository and an unchanged second sync. `TestGitHubEmptyRepositoryFailures`covers missing branches, unrelated ref conflicts and another computer winning-the initial write. `TestGitHubNotBackup` covers backend-specific recovery+the initial write. `TestGitHubLargeBackup` writes a 21 MB backup past a fixture whose Contents+API refuses it, as one commit, and reads it back by blob SHA.+`TestGitHubBranchMovedMeanwhile` covers a head that moved before the ref+update, and `TestGitHubEmptyRepositoryLargeBackup` a large first backup.+`TestGitHubNotBackup` covers backend-specific recoveryadvice when the repository holds an empty or foreign file.Go tests use `testenv` homes and HTTP fixtures. Browser tests use isolateddiff --git a/internal/davsync/github.go b/internal/davsync/github.goindex e593f5703..a750a89f1 100644--- a/internal/davsync/github.go+++ b/internal/davsync/github.go@@ -3,7 +3,9 @@ package davsyncimport ("bytes""context"+ "crypto/sha1""encoding/base64"+ "encoding/hex""encoding/json""errors""fmt"@@ -74,13 +76,15 @@ func newGitHub(c Config) (*github, error) {branch: branch, token: c.Password, client: &client}, nil}-func (g *github) send(ctx context.Context, method, apiPath string, body []byte, accept string, ref bool) (*http.Response, error) {+// send calls the repository's API; a non-empty ref reads at that branch or+// commit.+func (g *github) send(ctx context.Context, method, apiPath string, body []byte, accept string, ref string) (*http.Response, error) {u := &url.URL{Scheme: "https", Host: "api.github.com", Path: "/repos/" + g.repo}if apiPath != "" {u.Path += "/" + apiPath}- if ref {- q := url.Values{"ref": {g.branch}}+ if ref != "" {+ q := url.Values{"ref": {ref}}u.RawQuery = q.Encode()}req, err := http.NewRequestWithContext(ctx, method, u.String(), bytes.NewReader(body))@@ -129,7 +133,7 @@ func (g *github) prepare(ctx context.Context) error {if g.ready {return nil}- res, err := g.send(ctx, http.MethodGet, "", nil, "application/vnd.github+json", false)+ res, err := g.send(ctx, http.MethodGet, "", nil, "application/vnd.github+json", "")if err != nil {return err}@@ -157,7 +161,7 @@ func (g *github) prepare(ctx context.Context) error {if g.branch == "" {return errors.New("the GitHub repository has no default branch: create an initial commit first")}- res, err = g.send(ctx, http.MethodGet, "git/ref/heads/"+g.branch, nil, "application/vnd.github+json", false)+ res, err = g.send(ctx, http.MethodGet, "git/ref/heads/"+g.branch, nil, "application/vnd.github+json", "")if err != nil {return err}@@ -200,7 +204,14 @@ func (g *github) content(ctx context.Context, name string) (*githubContent, erroif err := g.prepare(ctx); err != nil {return nil, err}- res, err := g.send(ctx, http.MethodGet, "contents/"+g.at(name), nil, "application/vnd.github.object+json", true)+ return g.contentAt(ctx, name, g.branch)+}++// contentAt reads a file's metadata at a branch or commit. The object media+// type answers for files up to 100 MB; over 1 MB its content is empty and+// bytes downloads the blob instead.+func (g *github) contentAt(ctx context.Context, name, ref string) (*githubContent, error) {+ res, err := g.send(ctx, http.MethodGet, "contents/"+g.at(name), nil, "application/vnd.github.object+json", ref)if err != nil {return nil, err}@@ -238,7 +249,7 @@ func (g *github) bytes(ctx context.Context, c *githubContent) ([]byte, error) {}// The blob SHA pins the download to the metadata read above, even if// another computer commits a new backup while the download is starting.- res, err := g.send(ctx, http.MethodGet, "git/blobs/"+c.SHA, nil, "application/vnd.github.raw+json", false)+ res, err := g.send(ctx, http.MethodGet, "git/blobs/"+c.SHA, nil, "application/vnd.github.raw+json", "")if err != nil {return nil, err}@@ -266,6 +277,21 @@ func (g *github) get(ctx context.Context, have version) ([]byte, version, error)return b, v, err}+// errTooLarge is the Contents API refusing a file it won't process, which+// it does well under the 100 MB a blob can hold (a 19 MB backup passed and a+// larger one didn't, on Discord).+var errTooLarge = errors.New("GitHub's Contents API refused the file as too large")++// errBehind is a ref update that is no longer a fast-forward: another commit+// reached the branch after its head was read.+var errBehind = errors.New("the GitHub branch moved meanwhile")++// commit writes or removes one file in one commit. A repository with+// commits goes through the Git Data API (blob, tree, commit, ref), which+// takes files up to 100 MB, where the Contents API refuses large ones (HTTP+// 422 "the file is too large to be processed"). An empty repository has no+// branch to build on, so its first write goes through the Contents API,+// which creates the default branch.func (g *github) commit(ctx context.Context, name string, data []byte, sha string, remove bool) (version, error) {if len(data) > githubMaxFile {return version{}, errors.New("the GitHub sync file exceeds 64 MiB")@@ -273,6 +299,193 @@ func (g *github) commit(ctx context.Context, name string, data []byte, sha strinif err := g.prepare(ctx); err != nil {return version{}, err}+ if g.empty {+ v, err := g.contents(ctx, name, data, sha, remove)+ if !errors.Is(err, errTooLarge) {+ return v, err+ }+ // Too large for the Contents API: give the repository a first+ // commit of its own, then build the file's commit on it.+ if err := g.initialize(ctx); err != nil {+ return version{}, err+ }+ }+ blob := ""+ for try := 0; ; try++ {+ v, err := g.gitCommit(ctx, name, data, sha, remove, &blob)+ if !errors.Is(err, errBehind) {+ return v, err+ }+ // A commit that touched other files (another computer's usage)+ // is built on again; gitCommit returns errChanged if it touched+ // this file.+ if try == 3 {+ return version{}, errChanged+ }+ }+}++// githubJSON sends in as JSON and decodes a 2xx reply into out. Any other+// status is returned as the response, its body readable, for the caller to+// judge.+func (g *github) githubJSON(ctx context.Context, method, apiPath string, in, out any) (*http.Response, error) {+ var body []byte+ if in != nil {+ var err error+ if body, err = json.Marshal(in); err != nil {+ return nil, err+ }+ }+ res, err := g.send(ctx, method, apiPath, body, "application/vnd.github+json", "")+ if err != nil {+ return nil, err+ }+ if res.StatusCode < 200 || res.StatusCode > 299 {+ b, err := io.ReadAll(io.LimitReader(res.Body, 64<<10))+ res.Body.Close()+ if err != nil {+ return nil, err+ }+ res.Body = io.NopCloser(bytes.NewReader(b))+ return res, nil+ }+ defer res.Body.Close()+ if err := json.NewDecoder(io.LimitReader(res.Body, 1<<20)).Decode(out); err != nil {+ return nil, fmt.Errorf("reading GitHub's reply to %s %s: %w", method, apiPath, err)+ }+ return nil, nil+}++// gitCommit builds one commit on the branch's head that writes data (or+// removes the file) and moves the branch to it. The file must still be at+// sha in that head, so another computer's backup is never written over;+// the ref update without force refuses a head that moved after it was read.+// blob carries the uploaded blob's SHA across retries.+func (g *github) gitCommit(ctx context.Context, name string, data []byte, sha string, remove bool, blob *string) (version, error) {+ var ref struct {+ Object struct {+ SHA string `json:"sha"`+ } `json:"object"`+ }+ if res, err := g.githubJSON(ctx, http.MethodGet, "git/ref/heads/"+g.branch, nil, &ref); err != nil {+ return version{}, err+ } else if res != nil {+ return version{}, githubError(res)+ }+ head := ref.Object.SHA+ if head == "" {+ return version{}, errors.New("GitHub did not return the branch's head commit")+ }+ c, err := g.contentAt(ctx, name, head)+ if err != nil {+ return version{}, err+ }+ if c != nil && c.SHA != sha || c == nil && sha != "" {+ return version{}, errChanged+ }+ if !remove && *blob == "" {+ var out struct {+ SHA string `json:"sha"`+ }+ in := map[string]string{"content": base64.StdEncoding.EncodeToString(data), "encoding": "base64"}+ if res, err := g.githubJSON(ctx, http.MethodPost, "git/blobs", in, &out); err != nil {+ return version{}, err+ } else if res != nil {+ return version{}, githubError(res)+ }+ if out.SHA != gitBlobSHA(data) {+ return version{}, errors.New("GitHub stored a different blob than the one sent")+ }+ *blob = out.SHA+ }+ var commit struct {+ Tree struct {+ SHA string `json:"sha"`+ } `json:"tree"`+ }+ if res, err := g.githubJSON(ctx, http.MethodGet, "git/commits/"+head, nil, &commit); err != nil {+ return version{}, err+ } else if res != nil {+ return version{}, githubError(res)+ }+ if commit.Tree.SHA == "" {+ return version{}, errors.New("GitHub did not return the head commit's tree")+ }+ // A null sha removes the path from the tree.+ type entry struct {+ Path string `json:"path"`+ Mode string `json:"mode"`+ Type string `json:"type"`+ SHA *string `json:"sha"`+ }+ e := entry{Path: g.at(name), Mode: "100644", Type: "blob"}+ message := "magpie: remove " + name+ if !remove {+ e.SHA, message = blob, "magpie: sync "+name+ }+ var tree, made struct {+ SHA string `json:"sha"`+ }+ if res, err := g.githubJSON(ctx, http.MethodPost, "git/trees", map[string]any{"base_tree": commit.Tree.SHA, "tree": []entry{e}}, &tree); err != nil {+ return version{}, err+ } else if res != nil {+ return version{}, githubError(res)+ }+ if res, err := g.githubJSON(ctx, http.MethodPost, "git/commits", map[string]any{"message": message, "tree": tree.SHA, "parents": []string{head}}, &made); err != nil {+ return version{}, err+ } else if res != nil {+ return version{}, githubError(res)+ }+ if made.SHA == "" {+ return version{}, errors.New("GitHub did not return the new commit")+ }+ var moved json.RawMessage+ res, err := g.githubJSON(ctx, http.MethodPatch, "git/refs/heads/"+g.branch, map[string]any{"sha": made.SHA, "force": false}, &moved)+ if err != nil {+ return version{}, err+ }+ if res != nil {+ if res.StatusCode == http.StatusUnprocessableEntity {+ b, _ := io.ReadAll(res.Body)+ if strings.Contains(strings.ToLower(string(b)), "fast forward") {+ return version{}, errBehind+ }+ res.Body = io.NopCloser(bytes.NewReader(b))+ }+ return version{}, githubError(res)+ }+ g.empty = false+ if remove {+ return version{}, nil+ }+ return version{ETag: *blob}, nil+}++// gitBlobSHA is the SHA git names a blob holding b by: the version GitHub+// reports for the file.+func gitBlobSHA(b []byte) string {+ h := sha1.New()+ fmt.Fprintf(h, "blob %d\x00", len(b))+ h.Write(b)+ return hex.EncodeToString(h.Sum(nil))+}++// initialize gives an empty repository its first commit, a short note in+// magpie's folder, so a backup too large for the Contents API can be+// committed on top of it through the Git Data API. Another computer that+// initialized it first is just as good.+func (g *github) initialize(ctx context.Context) error {+ note := []byte("This folder holds magpie's encrypted sync files.\n")+ if _, err := g.contents(ctx, "README.md", note, "", false); err != nil && !errors.Is(err, errChanged) {+ return err+ }+ g.empty = false+ return nil+}++// contents writes through the Contents API, which only an empty repository's+// first write uses.+func (g *github) contents(ctx context.Context, name string, data []byte, sha string, remove bool) (version, error) {in := map[string]string{"message": "magpie: sync " + name, "content": base64.StdEncoding.EncodeToString(data)}// Omitting branch lets the first backup initialize an empty repository.if !g.empty {@@ -291,7 +504,7 @@ func (g *github) commit(ctx context.Context, name string, data []byte, sha strinif err != nil {return version{}, err}- res, err := g.send(ctx, method, "contents/"+g.at(name), body, "application/vnd.github+json", false)+ res, err := g.send(ctx, method, "contents/"+g.at(name), body, "application/vnd.github+json", "")if err != nil {return version{}, err}@@ -299,12 +512,19 @@ func (g *github) commit(ctx context.Context, name string, data []byte, sha strinif res.StatusCode == http.StatusConflict {return version{}, errChanged}+ if res.StatusCode == http.StatusRequestEntityTooLarge {+ return version{}, errTooLarge+ }if res.StatusCode == http.StatusUnprocessableEntity {- // A first write raced another computer creating the same file.body, _ := io.ReadAll(io.LimitReader(res.Body, 64<<10))+ // A first write raced another computer creating the same file.if strings.Contains(string(body), `\"sha\" wasn't supplied`) {return version{}, errChanged}+ var e struct{ Message string }+ if json.Unmarshal(body, &e) == nil && strings.Contains(strings.ToLower(e.Message), "too large") {+ return version{}, errTooLarge+ }res.Body = io.NopCloser(bytes.NewReader(body))}if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusCreated {@@ -332,7 +552,7 @@ func (g *github) list(ctx context.Context) (map[string]string, error) {if err := g.prepare(ctx); err != nil {return nil, err}- res, err := g.send(ctx, http.MethodGet, "contents/"+g.at(usageFolder), nil, "application/vnd.github+json", true)+ res, err := g.send(ctx, http.MethodGet, "contents/"+g.at(usageFolder), nil, "application/vnd.github+json", g.branch)if err != nil {return nil, err}diff --git a/internal/davsync/github_test.go b/internal/davsync/github_test.goindex 60313c03e..21fc39acf 100644--- a/internal/davsync/github_test.go+++ b/internal/davsync/github_test.go@@ -34,6 +34,62 @@ type githubFixture struct {initialBranch stringdefaultBranch stringpublic bool+ // The Git Data API's objects: a branch's head, blobs by SHA, trees+ // made on a base tree, and commits with their tree and parent.+ heads map[string]string+ blobs map[string][]byte+ trees map[string]fixtureTree+ commits map[string]fixtureCommit+ made int+ // contentsLimit is the largest file the Contents API's PUT takes;+ // GitHub answers 422 "too large" over its own (undocumented) limit.+ contentsLimit int+ // beforeRef runs, under the lock, before a ref update is judged: a+ // test commits there as another computer would.+ beforeRef func()+ // requests counts each method and path, to show which API wrote.+ requests []string+}++type fixtureTree struct {+ base string+ entries []struct {+ Path string+ SHA *string+ }+}++type fixtureCommit struct{ tree, parent string }++// head is the branch's head commit, made the first time it is asked for.+func (f *githubFixture) head(branch string) string {+ if f.heads[branch] == "" {+ f.advance(branch)+ }+ return f.heads[branch]+}++// advance commits on the branch outside the Git Data API (the Contents API,+// or another computer's commit set up by a test).+func (f *githubFixture) advance(branch string) {+ f.made+++ sha := fmt.Sprintf("%040x", f.made)+ f.commits[sha] = fixtureCommit{tree: "tree-" + sha, parent: f.heads[branch]}+ f.heads[branch] = sha+}++// branchAt is the branch a contents read's ref names: a branch, or a+// commit that is a branch's head now.+func (f *githubFixture) branchAt(ref string) (string, bool) {+ for b, h := range f.heads {+ if h == ref {+ return b, true+ }+ }+ if _, ok := f.commits[ref]; ok {+ return "", false // an older commit: the fixture keeps no old trees+ }+ return ref, true}type githubTransport struct {@@ -61,7 +117,8 @@ func githubSHA(b []byte) string {func newGithubFixture(t *testing.T) *githubFixture {t.Helper()- f := &githubFixture{objects: map[string][]byte{}, defaultBranch: "main"}+ f := &githubFixture{objects: map[string][]byte{}, defaultBranch: "main", heads: map[string]string{},+ blobs: map[string][]byte{}, trees: map[string]fixtureTree{}, commits: map[string]fixtureCommit{}, contentsLimit: 1 << 20}srv := httptest.NewServer(http.HandlerFunc(f.serve))t.Cleanup(srv.Close)target, _ := url.Parse(srv.URL)@@ -80,6 +137,7 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {json.NewEncoder(w).Encode(v)}fail := func(status int, message string) { send(status, map[string]string{"message": message}) }+ f.requests = append(f.requests, r.Method+" "+r.URL.Path)if r.Header.Get("Authorization") != "Bearer fixture-github-token" || r.Header.Get("X-GitHub-Api-Version") != "2022-11-28" {fail(401, "Bad credentials")return@@ -110,7 +168,10 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {fail(404, "Not Found")return}- send(200, map[string]string{"ref": "refs/heads/" + branch})+ send(200, map[string]any{"ref": "refs/heads/" + branch, "object": map[string]string{"type": "commit", "sha": f.head(branch)}})+ return+ }+ if f.serveGit(w, r, base, send, fail) {return}if strings.HasPrefix(r.URL.Path, base+"/git/blobs/") {@@ -134,7 +195,11 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {fail(404, "This repository is empty.")return}- branch := r.URL.Query().Get("ref")+ branch, ok := f.branchAt(r.URL.Query().Get("ref"))+ if !ok {+ fail(404, "No commit found for the ref")+ return+ }b, ok := f.objects[branch+"|"+path]if ok {c := githubContent{Type: "file", SHA: githubSHA(b), Size: int64(len(b)), Encoding: "base64", Content: base64.StdEncoding.EncodeToString(b)}@@ -178,6 +243,7 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {}if r.Method == http.MethodDelete {delete(f.objects, key)+ f.advance(in.Branch)f.writes++send(200, map[string]any{"content": nil})return@@ -187,7 +253,12 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {fail(422, "Invalid content")return}+ if len(b) > f.contentsLimit {+ fail(422, "Sorry, the file is too large to be processed. Consider creating/updating the file in a local clone and pushing it to GitHub.")+ return+ }f.objects[key] = b+ f.advance(in.Branch)f.writes++if f.empty {f.initialBranch = initialBranch@@ -200,6 +271,118 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {send(status, map[string]any{"content": githubContent{Type: "file", SHA: githubSHA(b), Size: int64(len(b))}})}+// serveGit answers the Git Data API as GitHub does: blobs, trees on a base+// tree, commits, and a ref update that refuses anything but a fast-forward.+func (f *githubFixture) serveGit(w http.ResponseWriter, r *http.Request, base string, send func(int, any), fail func(int, string)) bool {+ rest, ok := strings.CutPrefix(r.URL.Path, base+"/git/")+ if !ok || strings.HasPrefix(rest, "blobs/") && r.Method == http.MethodGet {+ return false+ }+ if f.empty {+ fail(409, "Git Repository is empty.")+ return true+ }+ switch {+ case rest == "blobs" && r.Method == http.MethodPost:+ var in struct{ Content, Encoding string }+ json.NewDecoder(r.Body).Decode(&in)+ b, err := base64.StdEncoding.DecodeString(in.Content)+ if err != nil || in.Encoding != "base64" {+ fail(422, "Invalid blob")+ return true+ }+ f.blobs[githubSHA(b)] = b+ send(201, map[string]string{"sha": githubSHA(b)})+ case strings.HasPrefix(rest, "commits/") && r.Method == http.MethodGet:+ c, ok := f.commits[strings.TrimPrefix(rest, "commits/")]+ if !ok {+ fail(404, "Not Found")+ return true+ }+ send(200, map[string]any{"tree": map[string]string{"sha": c.tree}})+ case rest == "trees" && r.Method == http.MethodPost:+ var in struct {+ BaseTree string `json:"base_tree"`+ Tree []struct {+ Path, Mode, Type string+ SHA *string+ }+ }+ raw, _ := io.ReadAll(r.Body)+ if json.Unmarshal(raw, &in) != nil || in.BaseTree == "" || len(in.Tree) == 0 || !strings.Contains(string(raw), `"sha":`) {+ fail(422, "Invalid tree")+ return true+ }+ var t fixtureTree+ t.base = in.BaseTree+ for _, e := range in.Tree {+ if e.Mode != "100644" || e.Type != "blob" || e.SHA != nil && f.blobs[*e.SHA] == nil {+ fail(422, "Invalid tree entry")+ return true+ }+ t.entries = append(t.entries, struct {+ Path string+ SHA *string+ }{e.Path, e.SHA})+ }+ f.made+++ sha := fmt.Sprintf("%040x", f.made)+ f.trees[sha] = t+ send(201, map[string]string{"sha": sha})+ case rest == "commits" && r.Method == http.MethodPost:+ var in struct {+ Message, Tree string+ Parents []string+ }+ json.NewDecoder(r.Body).Decode(&in)+ if _, ok := f.trees[in.Tree]; !ok || in.Message == "" || len(in.Parents) != 1 {+ fail(422, "Invalid commit")+ return true+ }+ f.made+++ sha := fmt.Sprintf("%040x", f.made)+ f.commits[sha] = fixtureCommit{tree: in.Tree, parent: in.Parents[0]}+ send(201, map[string]string{"sha": sha})+ case strings.HasPrefix(rest, "refs/heads/") && r.Method == http.MethodPatch:+ branch := strings.TrimPrefix(rest, "refs/heads/")+ var in struct {+ SHA string+ Force *bool+ }+ json.NewDecoder(r.Body).Decode(&in)+ if f.beforeRef != nil {+ f.beforeRef()+ }+ c, ok := f.commits[in.SHA]+ if f.heads[branch] == "" || !ok {+ fail(422, "Reference does not exist")+ return true+ }+ if in.Force == nil || *in.Force || c.parent != f.heads[branch] {+ fail(422, "Update is not a fast forward")+ return true+ }+ t := f.trees[c.tree]+ if t.base != f.commits[c.parent].tree {+ fail(422, "the fixture builds only on the parent's tree")+ return true+ }+ for _, e := range t.entries {+ if e.SHA == nil {+ delete(f.objects, branch+"|"+e.Path)+ } else {+ f.objects[branch+"|"+e.Path] = f.blobs[*e.SHA]+ }+ }+ f.heads[branch] = in.SHA+ f.writes+++ send(200, map[string]any{"ref": "refs/heads/" + branch, "object": map[string]string{"sha": in.SHA}})+ default:+ fail(404, "Not Found")+ }+ return true+}+func githubConfig() Config {return Config{URL: "github://alice/private/team", Password: "fixture-github-token",Passphrase: "fixture-independent-passphrase", Keys: true, Agents: true}@@ -314,6 +497,9 @@ func TestGitHubRemote(t *testing.T) {if err := g.remove(ctx, "computer-2026-10-06.magpie-usage"); err != nil {t.Fatal(err)}+ if _, ok := f.objects["sync/settings|team/magpie/usage/computer-2026-10-06.magpie-usage"]; ok {+ t.Fatal("the usage file was not removed")+ }if err := g.remove(ctx, "computer-2026-10-06.magpie-usage"); err != nil {t.Fatalf("already removed: %v", err)}@@ -576,3 +762,135 @@ func TestGitHubPublicRepositoryRefused(t *testing.T) {t.Fatalf("a public repository got the backup: writes %d, files %d", f.writes, len(f.objects))}}++// Zhang-Wei's backup (102 skills) passed the Contents API at about 19 MB+// and then grew past it: GitHub answered 422 "Sorry, the file is too large+// to be processed". A backup goes up as a blob, one commit per update, and+// is read back by its blob SHA.+func TestGitHubLargeBackup(t *testing.T) {+ f := newGithubFixture(t)+ f.contentsLimit = 19 << 20+ g, err := newGitHub(githubConfig())+ if err != nil {+ t.Fatal(err)+ }+ ctx := context.Background()+ small := []byte("an earlier sealed backup")+ v, err := g.put(ctx, small, "")+ if err != nil {+ t.Fatal(err)+ }+ large := make([]byte, 21<<20)+ for i := range large {+ large[i] = byte(i*7919 + i>>11)+ }+ before := f.writes+ v, err = g.put(ctx, large, v.ETag)+ if err != nil {+ t.Fatalf("a 21 MB backup: %v", err)+ }+ if v.ETag != githubSHA(large) || f.writes != before+1 {+ t.Fatalf("one commit per update, versioned by the blob: %+v, commits %d", v, f.writes-before)+ }+ if !bytesEqual(f.objects["main|team/magpie/"+file], large) {+ t.Fatal("the branch doesn't hold the large backup")+ }+ b, got, err := g.get(ctx, version{})+ if err != nil || got != v || !bytesEqual(b, large) {+ t.Fatalf("reading the 21 MB backup back: size %d, %+v, %v", len(b), got, err)+ }+ if !slices.Contains(f.requests, "GET /repos/alice/private/git/blobs/"+v.ETag) {+ t.Fatal("a file over 1 MB must be downloaded by its blob SHA")+ }+ for _, r := range f.requests {+ if strings.HasPrefix(r, "PUT ") || strings.HasPrefix(r, "DELETE ") {+ t.Fatalf("a repository with commits was written through the Contents API: %s", r)+ }+ }+ if _, err := g.put(ctx, small, "old-sha"); !errors.Is(err, errChanged) || !bytesEqual(f.objects["main|team/magpie/"+file], large) {+ t.Fatalf("a stale version must not replace the backup: %v", err)+ }+}++func bytesEqual(a, b []byte) bool { return string(a) == string(b) }++// The ref update refuses a head that moved after it was read. A commit of+// another file is built on again; one that changed the backup is the+// normal merge.+func TestGitHubBranchMovedMeanwhile(t *testing.T) {+ setup := func(t *testing.T) (*githubFixture, *github, version) {+ f := newGithubFixture(t)+ g, err := newGitHub(githubConfig())+ if err != nil {+ t.Fatal(err)+ }+ v, err := g.put(context.Background(), []byte("first"), "")+ if err != nil {+ t.Fatal(err)+ }+ return f, g, v+ }+ t.Run("another file", func(t *testing.T) {+ f, g, v := setup(t)+ f.beforeRef = func() {+ f.beforeRef = nil+ f.objects["main|team/magpie/usage/other.magpie-usage"] = []byte("other computer")+ f.advance("main")+ }+ if _, err := g.put(context.Background(), []byte("second"), v.ETag); err != nil {+ t.Fatal(err)+ }+ if string(f.objects["main|team/magpie/"+file]) != "second" || string(f.objects["main|team/magpie/usage/other.magpie-usage"]) != "other computer" {+ t.Fatal("both commits must be on the branch")+ }+ })+ t.Run("the backup", func(t *testing.T) {+ f, g, v := setup(t)+ f.beforeRef = func() {+ f.beforeRef = nil+ f.objects["main|team/magpie/"+file] = []byte("other computer's backup")+ f.advance("main")+ }+ if _, err := g.put(context.Background(), []byte("second"), v.ETag); !errors.Is(err, errChanged) {+ t.Fatalf("another computer's backup must go through the merge: %v", err)+ }+ if string(f.objects["main|team/magpie/"+file]) != "other computer's backup" {+ t.Fatal("another computer's backup was written over")+ }+ })+ t.Run("branch deleted", func(t *testing.T) {+ f, g, v := setup(t)+ f.beforeRef = func() { delete(f.heads, "main") }+ _, err := g.put(context.Background(), []byte("second"), v.ETag)+ if err == nil || errors.Is(err, errChanged) || !strings.Contains(err.Error(), "Reference does not exist") {+ t.Fatalf("a deleted branch must be an error, not a merge: %v", err)+ }+ })+}++// An empty repository has no branch for the Git Data API to build on. A+// first backup too large for the Contents API follows a short note that+// creates the default branch.+func TestGitHubEmptyRepositoryLargeBackup(t *testing.T) {+ f := newGithubFixture(t)+ f.empty, f.contentsLimit = true, 1<<10+ g, err := newGitHub(githubConfig())+ if err != nil {+ t.Fatal(err)+ }+ ctx := context.Background()+ if _, _, err := g.get(ctx, version{}); err != nil {+ t.Fatal(err)+ }+ large := []byte(strings.Repeat("sealed ", 1000))+ v, err := g.put(ctx, large, "")+ if err != nil {+ t.Fatalf("a large first backup in an empty repository: %v", err)+ }+ if f.writes != 2 || len(f.objects["main|team/magpie/README.md"]) == 0 || string(f.objects["main|team/magpie/"+file]) != string(large) {+ t.Fatalf("a note, then the backup: commits %d, files %v", f.writes, len(f.objects))+ }+ if b, got, err := g.get(ctx, version{}); err != nil || got != v || string(b) != string(large) {+ t.Fatalf("reading it back: %v", err)+ }+}