Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

davsync: a GitHub backup too large for the Contents API goes up as a blob (Zhang-Wei, Discord)

yetone93b87d00044d
diff --git a/docs/subsystems/sync-backup.md b/docs/subsystems/sync-backup.md
index cbf52fc02..ab2531b06 100644
--- a/docs/subsystems/sync-backup.md
+++ b/docs/subsystems/sync-backup.md
@@ -12,8 +12,8 @@ clone a working tree or synchronize arbitrary files.
versions, notices and undo file in `sync-state.json`. These files live beside
magpie's settings. Secrets are local, never in the status response.
- The [`remote`](../../internal/davsync/dav.go) interface reads a version and
- conditionally replaces it. WebDAV and S3 use ETags; GitHub uses the blob SHA
- required by its Contents API. [`github.go`](../../internal/davsync/github.go)
+ conditionally replaces it. WebDAV and S3 use ETags; GitHub uses the file's
+ blob SHA. [`github.go`](../../internal/davsync/github.go)
interprets `github://owner/repo[/prefix]`, with `Config.Branch` selecting a
branch or using the repository's default.
- [`backup.go`](../../internal/backup/backup.go) owns bundle contents and
@@ -97,12 +97,23 @@ repository can keep the token. The branch is part of the sync state key.
repository read doesn't say `"private": true`, since a public history hands
every sealed version to anyone to guess the passphrase against
(`TestGitHubPublicRepositoryRefused`). In an empty repository, the first encrypted
- backup initializes its default branch through the Contents API. A selected
+ backup initializes its default branch through the Contents API. When that
+ API refuses it as too large, a short `README.md` in magpie's folder makes
+ the first commit and the backup follows as the second. A selected
branch other than the default must already exist. The token needs repository
Contents read and write.
- Backups are `<prefix>/magpie/magpie.magpie-backup`; optional usage files
- live beside them under `usage/`. Changes create commits, and unchanged
- setup does not create another backup commit.
+ live beside them under `usage/`. Each write or removal is one commit, and
+ unchanged setup does not create another backup commit.
+- A repository with commits is written through the Git Data API: a blob
+ (up to 100 MB), a tree on the head commit's tree, a commit with the head
+ as its parent, and a ref update without force. The Contents API's PUT
+ refuses files well under that with 422 "the file is too large to be
+ processed" (a 102-skill backup past ~19 MB, Discord). The file must still
+ be at the version sync read, checked at the head commit, before the
+ commit is built. A ref update that is not a fast-forward is built again on
+ the new head, up to three times; if the new head changed the file it is
+ `errChanged`. A branch deleted meanwhile is an error, never created.
- Tokens are separate from the library's GitHub token, sent only to
`https://api.github.com`, and must differ from the encryption passphrase.
Repository history retains older encrypted versions.
@@ -115,7 +126,8 @@ repository can keep the token. The branch is part of the sync state key.
- Large files are downloaded by blob SHA, preserving the version that was
read. Sync files are limited to 64 MiB. Usage listing fails at the Contents
API's 1,000-file limit rather than treating a truncated list as complete.
-- A stale SHA or a racing initial creation returns `errChanged`. Other 422
+- A stale SHA, a backup another commit changed, or a racing initial
+ creation returns `errChanged`. Other 422
responses stay actionable errors. Authentication, permission and branch
failures remain distinct from missing files. Rate limits use sync backoff.
- Request archival remains an S3-only feature; its objects are separate from
@@ -132,7 +144,11 @@ node --test internal/gui/tests/github-sync.test.cjs internal/gui/tests/s3-sync.t
`TestGitHubEmptyRepository` covers the first encrypted backup initializing an
empty repository and an unchanged second sync. `TestGitHubEmptyRepositoryFailures`
covers missing branches, unrelated ref conflicts and another computer winning
-the initial write. `TestGitHubNotBackup` covers backend-specific recovery
+the initial write. `TestGitHubLargeBackup` writes a 21 MB backup past a fixture whose Contents
+API refuses it, as one commit, and reads it back by blob SHA.
+`TestGitHubBranchMovedMeanwhile` covers a head that moved before the ref
+update, and `TestGitHubEmptyRepositoryLargeBackup` a large first backup.
+`TestGitHubNotBackup` covers backend-specific recovery
advice when the repository holds an empty or foreign file.
Go tests use `testenv` homes and HTTP fixtures. Browser tests use isolated
diff --git a/internal/davsync/github.go b/internal/davsync/github.go
index e593f5703..a750a89f1 100644
--- a/internal/davsync/github.go
+++ b/internal/davsync/github.go
@@ -3,7 +3,9 @@ package davsync
import (
"bytes"
"context"
+ "crypto/sha1"
"encoding/base64"
+ "encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -74,13 +76,15 @@ func newGitHub(c Config) (*github, error) {
branch: branch, token: c.Password, client: &client}, nil
}
-func (g *github) send(ctx context.Context, method, apiPath string, body []byte, accept string, ref bool) (*http.Response, error) {
+// send calls the repository's API; a non-empty ref reads at that branch or
+// commit.
+func (g *github) send(ctx context.Context, method, apiPath string, body []byte, accept string, ref string) (*http.Response, error) {
u := &url.URL{Scheme: "https", Host: "api.github.com", Path: "/repos/" + g.repo}
if apiPath != "" {
u.Path += "/" + apiPath
}
- if ref {
- q := url.Values{"ref": {g.branch}}
+ if ref != "" {
+ q := url.Values{"ref": {ref}}
u.RawQuery = q.Encode()
}
req, err := http.NewRequestWithContext(ctx, method, u.String(), bytes.NewReader(body))
@@ -129,7 +133,7 @@ func (g *github) prepare(ctx context.Context) error {
if g.ready {
return nil
}
- res, err := g.send(ctx, http.MethodGet, "", nil, "application/vnd.github+json", false)
+ res, err := g.send(ctx, http.MethodGet, "", nil, "application/vnd.github+json", "")
if err != nil {
return err
}
@@ -157,7 +161,7 @@ func (g *github) prepare(ctx context.Context) error {
if g.branch == "" {
return errors.New("the GitHub repository has no default branch: create an initial commit first")
}
- res, err = g.send(ctx, http.MethodGet, "git/ref/heads/"+g.branch, nil, "application/vnd.github+json", false)
+ res, err = g.send(ctx, http.MethodGet, "git/ref/heads/"+g.branch, nil, "application/vnd.github+json", "")
if err != nil {
return err
}
@@ -200,7 +204,14 @@ func (g *github) content(ctx context.Context, name string) (*githubContent, erro
if err := g.prepare(ctx); err != nil {
return nil, err
}
- res, err := g.send(ctx, http.MethodGet, "contents/"+g.at(name), nil, "application/vnd.github.object+json", true)
+ return g.contentAt(ctx, name, g.branch)
+}
+
+// contentAt reads a file's metadata at a branch or commit. The object media
+// type answers for files up to 100 MB; over 1 MB its content is empty and
+// bytes downloads the blob instead.
+func (g *github) contentAt(ctx context.Context, name, ref string) (*githubContent, error) {
+ res, err := g.send(ctx, http.MethodGet, "contents/"+g.at(name), nil, "application/vnd.github.object+json", ref)
if err != nil {
return nil, err
}
@@ -238,7 +249,7 @@ func (g *github) bytes(ctx context.Context, c *githubContent) ([]byte, error) {
}
// The blob SHA pins the download to the metadata read above, even if
// another computer commits a new backup while the download is starting.
- res, err := g.send(ctx, http.MethodGet, "git/blobs/"+c.SHA, nil, "application/vnd.github.raw+json", false)
+ res, err := g.send(ctx, http.MethodGet, "git/blobs/"+c.SHA, nil, "application/vnd.github.raw+json", "")
if err != nil {
return nil, err
}
@@ -266,6 +277,21 @@ func (g *github) get(ctx context.Context, have version) ([]byte, version, error)
return b, v, err
}
+// errTooLarge is the Contents API refusing a file it won't process, which
+// it does well under the 100 MB a blob can hold (a 19 MB backup passed and a
+// larger one didn't, on Discord).
+var errTooLarge = errors.New("GitHub's Contents API refused the file as too large")
+
+// errBehind is a ref update that is no longer a fast-forward: another commit
+// reached the branch after its head was read.
+var errBehind = errors.New("the GitHub branch moved meanwhile")
+
+// commit writes or removes one file in one commit. A repository with
+// commits goes through the Git Data API (blob, tree, commit, ref), which
+// takes files up to 100 MB, where the Contents API refuses large ones (HTTP
+// 422 "the file is too large to be processed"). An empty repository has no
+// branch to build on, so its first write goes through the Contents API,
+// which creates the default branch.
func (g *github) commit(ctx context.Context, name string, data []byte, sha string, remove bool) (version, error) {
if len(data) > githubMaxFile {
return version{}, errors.New("the GitHub sync file exceeds 64 MiB")
@@ -273,6 +299,193 @@ func (g *github) commit(ctx context.Context, name string, data []byte, sha strin
if err := g.prepare(ctx); err != nil {
return version{}, err
}
+ if g.empty {
+ v, err := g.contents(ctx, name, data, sha, remove)
+ if !errors.Is(err, errTooLarge) {
+ return v, err
+ }
+ // Too large for the Contents API: give the repository a first
+ // commit of its own, then build the file's commit on it.
+ if err := g.initialize(ctx); err != nil {
+ return version{}, err
+ }
+ }
+ blob := ""
+ for try := 0; ; try++ {
+ v, err := g.gitCommit(ctx, name, data, sha, remove, &blob)
+ if !errors.Is(err, errBehind) {
+ return v, err
+ }
+ // A commit that touched other files (another computer's usage)
+ // is built on again; gitCommit returns errChanged if it touched
+ // this file.
+ if try == 3 {
+ return version{}, errChanged
+ }
+ }
+}
+
+// githubJSON sends in as JSON and decodes a 2xx reply into out. Any other
+// status is returned as the response, its body readable, for the caller to
+// judge.
+func (g *github) githubJSON(ctx context.Context, method, apiPath string, in, out any) (*http.Response, error) {
+ var body []byte
+ if in != nil {
+ var err error
+ if body, err = json.Marshal(in); err != nil {
+ return nil, err
+ }
+ }
+ res, err := g.send(ctx, method, apiPath, body, "application/vnd.github+json", "")
+ if err != nil {
+ return nil, err
+ }
+ if res.StatusCode < 200 || res.StatusCode > 299 {
+ b, err := io.ReadAll(io.LimitReader(res.Body, 64<<10))
+ res.Body.Close()
+ if err != nil {
+ return nil, err
+ }
+ res.Body = io.NopCloser(bytes.NewReader(b))
+ return res, nil
+ }
+ defer res.Body.Close()
+ if err := json.NewDecoder(io.LimitReader(res.Body, 1<<20)).Decode(out); err != nil {
+ return nil, fmt.Errorf("reading GitHub's reply to %s %s: %w", method, apiPath, err)
+ }
+ return nil, nil
+}
+
+// gitCommit builds one commit on the branch's head that writes data (or
+// removes the file) and moves the branch to it. The file must still be at
+// sha in that head, so another computer's backup is never written over;
+// the ref update without force refuses a head that moved after it was read.
+// blob carries the uploaded blob's SHA across retries.
+func (g *github) gitCommit(ctx context.Context, name string, data []byte, sha string, remove bool, blob *string) (version, error) {
+ var ref struct {
+ Object struct {
+ SHA string `json:"sha"`
+ } `json:"object"`
+ }
+ if res, err := g.githubJSON(ctx, http.MethodGet, "git/ref/heads/"+g.branch, nil, &ref); err != nil {
+ return version{}, err
+ } else if res != nil {
+ return version{}, githubError(res)
+ }
+ head := ref.Object.SHA
+ if head == "" {
+ return version{}, errors.New("GitHub did not return the branch's head commit")
+ }
+ c, err := g.contentAt(ctx, name, head)
+ if err != nil {
+ return version{}, err
+ }
+ if c != nil && c.SHA != sha || c == nil && sha != "" {
+ return version{}, errChanged
+ }
+ if !remove && *blob == "" {
+ var out struct {
+ SHA string `json:"sha"`
+ }
+ in := map[string]string{"content": base64.StdEncoding.EncodeToString(data), "encoding": "base64"}
+ if res, err := g.githubJSON(ctx, http.MethodPost, "git/blobs", in, &out); err != nil {
+ return version{}, err
+ } else if res != nil {
+ return version{}, githubError(res)
+ }
+ if out.SHA != gitBlobSHA(data) {
+ return version{}, errors.New("GitHub stored a different blob than the one sent")
+ }
+ *blob = out.SHA
+ }
+ var commit struct {
+ Tree struct {
+ SHA string `json:"sha"`
+ } `json:"tree"`
+ }
+ if res, err := g.githubJSON(ctx, http.MethodGet, "git/commits/"+head, nil, &commit); err != nil {
+ return version{}, err
+ } else if res != nil {
+ return version{}, githubError(res)
+ }
+ if commit.Tree.SHA == "" {
+ return version{}, errors.New("GitHub did not return the head commit's tree")
+ }
+ // A null sha removes the path from the tree.
+ type entry struct {
+ Path string `json:"path"`
+ Mode string `json:"mode"`
+ Type string `json:"type"`
+ SHA *string `json:"sha"`
+ }
+ e := entry{Path: g.at(name), Mode: "100644", Type: "blob"}
+ message := "magpie: remove " + name
+ if !remove {
+ e.SHA, message = blob, "magpie: sync "+name
+ }
+ var tree, made struct {
+ SHA string `json:"sha"`
+ }
+ if res, err := g.githubJSON(ctx, http.MethodPost, "git/trees", map[string]any{"base_tree": commit.Tree.SHA, "tree": []entry{e}}, &tree); err != nil {
+ return version{}, err
+ } else if res != nil {
+ return version{}, githubError(res)
+ }
+ if res, err := g.githubJSON(ctx, http.MethodPost, "git/commits", map[string]any{"message": message, "tree": tree.SHA, "parents": []string{head}}, &made); err != nil {
+ return version{}, err
+ } else if res != nil {
+ return version{}, githubError(res)
+ }
+ if made.SHA == "" {
+ return version{}, errors.New("GitHub did not return the new commit")
+ }
+ var moved json.RawMessage
+ res, err := g.githubJSON(ctx, http.MethodPatch, "git/refs/heads/"+g.branch, map[string]any{"sha": made.SHA, "force": false}, &moved)
+ if err != nil {
+ return version{}, err
+ }
+ if res != nil {
+ if res.StatusCode == http.StatusUnprocessableEntity {
+ b, _ := io.ReadAll(res.Body)
+ if strings.Contains(strings.ToLower(string(b)), "fast forward") {
+ return version{}, errBehind
+ }
+ res.Body = io.NopCloser(bytes.NewReader(b))
+ }
+ return version{}, githubError(res)
+ }
+ g.empty = false
+ if remove {
+ return version{}, nil
+ }
+ return version{ETag: *blob}, nil
+}
+
+// gitBlobSHA is the SHA git names a blob holding b by: the version GitHub
+// reports for the file.
+func gitBlobSHA(b []byte) string {
+ h := sha1.New()
+ fmt.Fprintf(h, "blob %d\x00", len(b))
+ h.Write(b)
+ return hex.EncodeToString(h.Sum(nil))
+}
+
+// initialize gives an empty repository its first commit, a short note in
+// magpie's folder, so a backup too large for the Contents API can be
+// committed on top of it through the Git Data API. Another computer that
+// initialized it first is just as good.
+func (g *github) initialize(ctx context.Context) error {
+ note := []byte("This folder holds magpie's encrypted sync files.\n")
+ if _, err := g.contents(ctx, "README.md", note, "", false); err != nil && !errors.Is(err, errChanged) {
+ return err
+ }
+ g.empty = false
+ return nil
+}
+
+// contents writes through the Contents API, which only an empty repository's
+// first write uses.
+func (g *github) contents(ctx context.Context, name string, data []byte, sha string, remove bool) (version, error) {
in := map[string]string{"message": "magpie: sync " + name, "content": base64.StdEncoding.EncodeToString(data)}
// Omitting branch lets the first backup initialize an empty repository.
if !g.empty {
@@ -291,7 +504,7 @@ func (g *github) commit(ctx context.Context, name string, data []byte, sha strin
if err != nil {
return version{}, err
}
- res, err := g.send(ctx, method, "contents/"+g.at(name), body, "application/vnd.github+json", false)
+ res, err := g.send(ctx, method, "contents/"+g.at(name), body, "application/vnd.github+json", "")
if err != nil {
return version{}, err
}
@@ -299,12 +512,19 @@ func (g *github) commit(ctx context.Context, name string, data []byte, sha strin
if res.StatusCode == http.StatusConflict {
return version{}, errChanged
}
+ if res.StatusCode == http.StatusRequestEntityTooLarge {
+ return version{}, errTooLarge
+ }
if res.StatusCode == http.StatusUnprocessableEntity {
- // A first write raced another computer creating the same file.
body, _ := io.ReadAll(io.LimitReader(res.Body, 64<<10))
+ // A first write raced another computer creating the same file.
if strings.Contains(string(body), `\"sha\" wasn't supplied`) {
return version{}, errChanged
}
+ var e struct{ Message string }
+ if json.Unmarshal(body, &e) == nil && strings.Contains(strings.ToLower(e.Message), "too large") {
+ return version{}, errTooLarge
+ }
res.Body = io.NopCloser(bytes.NewReader(body))
}
if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusCreated {
@@ -332,7 +552,7 @@ func (g *github) list(ctx context.Context) (map[string]string, error) {
if err := g.prepare(ctx); err != nil {
return nil, err
}
- res, err := g.send(ctx, http.MethodGet, "contents/"+g.at(usageFolder), nil, "application/vnd.github+json", true)
+ res, err := g.send(ctx, http.MethodGet, "contents/"+g.at(usageFolder), nil, "application/vnd.github+json", g.branch)
if err != nil {
return nil, err
}
diff --git a/internal/davsync/github_test.go b/internal/davsync/github_test.go
index 60313c03e..21fc39acf 100644
--- a/internal/davsync/github_test.go
+++ b/internal/davsync/github_test.go
@@ -34,6 +34,62 @@ type githubFixture struct {
initialBranch string
defaultBranch string
public bool
+ // The Git Data API's objects: a branch's head, blobs by SHA, trees
+ // made on a base tree, and commits with their tree and parent.
+ heads map[string]string
+ blobs map[string][]byte
+ trees map[string]fixtureTree
+ commits map[string]fixtureCommit
+ made int
+ // contentsLimit is the largest file the Contents API's PUT takes;
+ // GitHub answers 422 "too large" over its own (undocumented) limit.
+ contentsLimit int
+ // beforeRef runs, under the lock, before a ref update is judged: a
+ // test commits there as another computer would.
+ beforeRef func()
+ // requests counts each method and path, to show which API wrote.
+ requests []string
+}
+
+type fixtureTree struct {
+ base string
+ entries []struct {
+ Path string
+ SHA *string
+ }
+}
+
+type fixtureCommit struct{ tree, parent string }
+
+// head is the branch's head commit, made the first time it is asked for.
+func (f *githubFixture) head(branch string) string {
+ if f.heads[branch] == "" {
+ f.advance(branch)
+ }
+ return f.heads[branch]
+}
+
+// advance commits on the branch outside the Git Data API (the Contents API,
+// or another computer's commit set up by a test).
+func (f *githubFixture) advance(branch string) {
+ f.made++
+ sha := fmt.Sprintf("%040x", f.made)
+ f.commits[sha] = fixtureCommit{tree: "tree-" + sha, parent: f.heads[branch]}
+ f.heads[branch] = sha
+}
+
+// branchAt is the branch a contents read's ref names: a branch, or a
+// commit that is a branch's head now.
+func (f *githubFixture) branchAt(ref string) (string, bool) {
+ for b, h := range f.heads {
+ if h == ref {
+ return b, true
+ }
+ }
+ if _, ok := f.commits[ref]; ok {
+ return "", false // an older commit: the fixture keeps no old trees
+ }
+ return ref, true
}
type githubTransport struct {
@@ -61,7 +117,8 @@ func githubSHA(b []byte) string {
func newGithubFixture(t *testing.T) *githubFixture {
t.Helper()
- f := &githubFixture{objects: map[string][]byte{}, defaultBranch: "main"}
+ f := &githubFixture{objects: map[string][]byte{}, defaultBranch: "main", heads: map[string]string{},
+ blobs: map[string][]byte{}, trees: map[string]fixtureTree{}, commits: map[string]fixtureCommit{}, contentsLimit: 1 << 20}
srv := httptest.NewServer(http.HandlerFunc(f.serve))
t.Cleanup(srv.Close)
target, _ := url.Parse(srv.URL)
@@ -80,6 +137,7 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(v)
}
fail := func(status int, message string) { send(status, map[string]string{"message": message}) }
+ f.requests = append(f.requests, r.Method+" "+r.URL.Path)
if r.Header.Get("Authorization") != "Bearer fixture-github-token" || r.Header.Get("X-GitHub-Api-Version") != "2022-11-28" {
fail(401, "Bad credentials")
return
@@ -110,7 +168,10 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {
fail(404, "Not Found")
return
}
- send(200, map[string]string{"ref": "refs/heads/" + branch})
+ send(200, map[string]any{"ref": "refs/heads/" + branch, "object": map[string]string{"type": "commit", "sha": f.head(branch)}})
+ return
+ }
+ if f.serveGit(w, r, base, send, fail) {
return
}
if strings.HasPrefix(r.URL.Path, base+"/git/blobs/") {
@@ -134,7 +195,11 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {
fail(404, "This repository is empty.")
return
}
- branch := r.URL.Query().Get("ref")
+ branch, ok := f.branchAt(r.URL.Query().Get("ref"))
+ if !ok {
+ fail(404, "No commit found for the ref")
+ return
+ }
b, ok := f.objects[branch+"|"+path]
if ok {
c := githubContent{Type: "file", SHA: githubSHA(b), Size: int64(len(b)), Encoding: "base64", Content: base64.StdEncoding.EncodeToString(b)}
@@ -178,6 +243,7 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {
}
if r.Method == http.MethodDelete {
delete(f.objects, key)
+ f.advance(in.Branch)
f.writes++
send(200, map[string]any{"content": nil})
return
@@ -187,7 +253,12 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {
fail(422, "Invalid content")
return
}
+ if len(b) > f.contentsLimit {
+ fail(422, "Sorry, the file is too large to be processed. Consider creating/updating the file in a local clone and pushing it to GitHub.")
+ return
+ }
f.objects[key] = b
+ f.advance(in.Branch)
f.writes++
if f.empty {
f.initialBranch = initialBranch
@@ -200,6 +271,118 @@ func (f *githubFixture) serve(w http.ResponseWriter, r *http.Request) {
send(status, map[string]any{"content": githubContent{Type: "file", SHA: githubSHA(b), Size: int64(len(b))}})
}
+// serveGit answers the Git Data API as GitHub does: blobs, trees on a base
+// tree, commits, and a ref update that refuses anything but a fast-forward.
+func (f *githubFixture) serveGit(w http.ResponseWriter, r *http.Request, base string, send func(int, any), fail func(int, string)) bool {
+ rest, ok := strings.CutPrefix(r.URL.Path, base+"/git/")
+ if !ok || strings.HasPrefix(rest, "blobs/") && r.Method == http.MethodGet {
+ return false
+ }
+ if f.empty {
+ fail(409, "Git Repository is empty.")
+ return true
+ }
+ switch {
+ case rest == "blobs" && r.Method == http.MethodPost:
+ var in struct{ Content, Encoding string }
+ json.NewDecoder(r.Body).Decode(&in)
+ b, err := base64.StdEncoding.DecodeString(in.Content)
+ if err != nil || in.Encoding != "base64" {
+ fail(422, "Invalid blob")
+ return true
+ }
+ f.blobs[githubSHA(b)] = b
+ send(201, map[string]string{"sha": githubSHA(b)})
+ case strings.HasPrefix(rest, "commits/") && r.Method == http.MethodGet:
+ c, ok := f.commits[strings.TrimPrefix(rest, "commits/")]
+ if !ok {
+ fail(404, "Not Found")
+ return true
+ }
+ send(200, map[string]any{"tree": map[string]string{"sha": c.tree}})
+ case rest == "trees" && r.Method == http.MethodPost:
+ var in struct {
+ BaseTree string `json:"base_tree"`
+ Tree []struct {
+ Path, Mode, Type string
+ SHA *string
+ }
+ }
+ raw, _ := io.ReadAll(r.Body)
+ if json.Unmarshal(raw, &in) != nil || in.BaseTree == "" || len(in.Tree) == 0 || !strings.Contains(string(raw), `"sha":`) {
+ fail(422, "Invalid tree")
+ return true
+ }
+ var t fixtureTree
+ t.base = in.BaseTree
+ for _, e := range in.Tree {
+ if e.Mode != "100644" || e.Type != "blob" || e.SHA != nil && f.blobs[*e.SHA] == nil {
+ fail(422, "Invalid tree entry")
+ return true
+ }
+ t.entries = append(t.entries, struct {
+ Path string
+ SHA *string
+ }{e.Path, e.SHA})
+ }
+ f.made++
+ sha := fmt.Sprintf("%040x", f.made)
+ f.trees[sha] = t
+ send(201, map[string]string{"sha": sha})
+ case rest == "commits" && r.Method == http.MethodPost:
+ var in struct {
+ Message, Tree string
+ Parents []string
+ }
+ json.NewDecoder(r.Body).Decode(&in)
+ if _, ok := f.trees[in.Tree]; !ok || in.Message == "" || len(in.Parents) != 1 {
+ fail(422, "Invalid commit")
+ return true
+ }
+ f.made++
+ sha := fmt.Sprintf("%040x", f.made)
+ f.commits[sha] = fixtureCommit{tree: in.Tree, parent: in.Parents[0]}
+ send(201, map[string]string{"sha": sha})
+ case strings.HasPrefix(rest, "refs/heads/") && r.Method == http.MethodPatch:
+ branch := strings.TrimPrefix(rest, "refs/heads/")
+ var in struct {
+ SHA string
+ Force *bool
+ }
+ json.NewDecoder(r.Body).Decode(&in)
+ if f.beforeRef != nil {
+ f.beforeRef()
+ }
+ c, ok := f.commits[in.SHA]
+ if f.heads[branch] == "" || !ok {
+ fail(422, "Reference does not exist")
+ return true
+ }
+ if in.Force == nil || *in.Force || c.parent != f.heads[branch] {
+ fail(422, "Update is not a fast forward")
+ return true
+ }
+ t := f.trees[c.tree]
+ if t.base != f.commits[c.parent].tree {
+ fail(422, "the fixture builds only on the parent's tree")
+ return true
+ }
+ for _, e := range t.entries {
+ if e.SHA == nil {
+ delete(f.objects, branch+"|"+e.Path)
+ } else {
+ f.objects[branch+"|"+e.Path] = f.blobs[*e.SHA]
+ }
+ }
+ f.heads[branch] = in.SHA
+ f.writes++
+ send(200, map[string]any{"ref": "refs/heads/" + branch, "object": map[string]string{"sha": in.SHA}})
+ default:
+ fail(404, "Not Found")
+ }
+ return true
+}
+
func githubConfig() Config {
return Config{URL: "github://alice/private/team", Password: "fixture-github-token",
Passphrase: "fixture-independent-passphrase", Keys: true, Agents: true}
@@ -314,6 +497,9 @@ func TestGitHubRemote(t *testing.T) {
if err := g.remove(ctx, "computer-2026-10-06.magpie-usage"); err != nil {
t.Fatal(err)
}
+ if _, ok := f.objects["sync/settings|team/magpie/usage/computer-2026-10-06.magpie-usage"]; ok {
+ t.Fatal("the usage file was not removed")
+ }
if err := g.remove(ctx, "computer-2026-10-06.magpie-usage"); err != nil {
t.Fatalf("already removed: %v", err)
}
@@ -576,3 +762,135 @@ func TestGitHubPublicRepositoryRefused(t *testing.T) {
t.Fatalf("a public repository got the backup: writes %d, files %d", f.writes, len(f.objects))
}
}
+
+// Zhang-Wei's backup (102 skills) passed the Contents API at about 19 MB
+// and then grew past it: GitHub answered 422 "Sorry, the file is too large
+// to be processed". A backup goes up as a blob, one commit per update, and
+// is read back by its blob SHA.
+func TestGitHubLargeBackup(t *testing.T) {
+ f := newGithubFixture(t)
+ f.contentsLimit = 19 << 20
+ g, err := newGitHub(githubConfig())
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+ small := []byte("an earlier sealed backup")
+ v, err := g.put(ctx, small, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ large := make([]byte, 21<<20)
+ for i := range large {
+ large[i] = byte(i*7919 + i>>11)
+ }
+ before := f.writes
+ v, err = g.put(ctx, large, v.ETag)
+ if err != nil {
+ t.Fatalf("a 21 MB backup: %v", err)
+ }
+ if v.ETag != githubSHA(large) || f.writes != before+1 {
+ t.Fatalf("one commit per update, versioned by the blob: %+v, commits %d", v, f.writes-before)
+ }
+ if !bytesEqual(f.objects["main|team/magpie/"+file], large) {
+ t.Fatal("the branch doesn't hold the large backup")
+ }
+ b, got, err := g.get(ctx, version{})
+ if err != nil || got != v || !bytesEqual(b, large) {
+ t.Fatalf("reading the 21 MB backup back: size %d, %+v, %v", len(b), got, err)
+ }
+ if !slices.Contains(f.requests, "GET /repos/alice/private/git/blobs/"+v.ETag) {
+ t.Fatal("a file over 1 MB must be downloaded by its blob SHA")
+ }
+ for _, r := range f.requests {
+ if strings.HasPrefix(r, "PUT ") || strings.HasPrefix(r, "DELETE ") {
+ t.Fatalf("a repository with commits was written through the Contents API: %s", r)
+ }
+ }
+ if _, err := g.put(ctx, small, "old-sha"); !errors.Is(err, errChanged) || !bytesEqual(f.objects["main|team/magpie/"+file], large) {
+ t.Fatalf("a stale version must not replace the backup: %v", err)
+ }
+}
+
+func bytesEqual(a, b []byte) bool { return string(a) == string(b) }
+
+// The ref update refuses a head that moved after it was read. A commit of
+// another file is built on again; one that changed the backup is the
+// normal merge.
+func TestGitHubBranchMovedMeanwhile(t *testing.T) {
+ setup := func(t *testing.T) (*githubFixture, *github, version) {
+ f := newGithubFixture(t)
+ g, err := newGitHub(githubConfig())
+ if err != nil {
+ t.Fatal(err)
+ }
+ v, err := g.put(context.Background(), []byte("first"), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return f, g, v
+ }
+ t.Run("another file", func(t *testing.T) {
+ f, g, v := setup(t)
+ f.beforeRef = func() {
+ f.beforeRef = nil
+ f.objects["main|team/magpie/usage/other.magpie-usage"] = []byte("other computer")
+ f.advance("main")
+ }
+ if _, err := g.put(context.Background(), []byte("second"), v.ETag); err != nil {
+ t.Fatal(err)
+ }
+ if string(f.objects["main|team/magpie/"+file]) != "second" || string(f.objects["main|team/magpie/usage/other.magpie-usage"]) != "other computer" {
+ t.Fatal("both commits must be on the branch")
+ }
+ })
+ t.Run("the backup", func(t *testing.T) {
+ f, g, v := setup(t)
+ f.beforeRef = func() {
+ f.beforeRef = nil
+ f.objects["main|team/magpie/"+file] = []byte("other computer's backup")
+ f.advance("main")
+ }
+ if _, err := g.put(context.Background(), []byte("second"), v.ETag); !errors.Is(err, errChanged) {
+ t.Fatalf("another computer's backup must go through the merge: %v", err)
+ }
+ if string(f.objects["main|team/magpie/"+file]) != "other computer's backup" {
+ t.Fatal("another computer's backup was written over")
+ }
+ })
+ t.Run("branch deleted", func(t *testing.T) {
+ f, g, v := setup(t)
+ f.beforeRef = func() { delete(f.heads, "main") }
+ _, err := g.put(context.Background(), []byte("second"), v.ETag)
+ if err == nil || errors.Is(err, errChanged) || !strings.Contains(err.Error(), "Reference does not exist") {
+ t.Fatalf("a deleted branch must be an error, not a merge: %v", err)
+ }
+ })
+}
+
+// An empty repository has no branch for the Git Data API to build on. A
+// first backup too large for the Contents API follows a short note that
+// creates the default branch.
+func TestGitHubEmptyRepositoryLargeBackup(t *testing.T) {
+ f := newGithubFixture(t)
+ f.empty, f.contentsLimit = true, 1<<10
+ g, err := newGitHub(githubConfig())
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+ if _, _, err := g.get(ctx, version{}); err != nil {
+ t.Fatal(err)
+ }
+ large := []byte(strings.Repeat("sealed ", 1000))
+ v, err := g.put(ctx, large, "")
+ if err != nil {
+ t.Fatalf("a large first backup in an empty repository: %v", err)
+ }
+ if f.writes != 2 || len(f.objects["main|team/magpie/README.md"]) == 0 || string(f.objects["main|team/magpie/"+file]) != string(large) {
+ t.Fatalf("a note, then the backup: commits %d, files %v", f.writes, len(f.objects))
+ }
+ if b, got, err := g.get(ctx, version{}); err != nil || got != v || string(b) != string(large) {
+ t.Fatalf("reading it back: %v", err)
+ }
+}